diff --git a/confgenerator/agentmetrics.go b/confgenerator/agentmetrics.go index 022c073821..0ca62f6a4c 100644 --- a/confgenerator/agentmetrics.go +++ b/confgenerator/agentmetrics.go @@ -22,9 +22,22 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel/ottl" + "github.com/GoogleCloudPlatform/ops-agent/internal/version" ) -// AgentSelfMetrics provides the agent.googleapis.com/agent/ metrics. +var ( + agentKind string = "ops-agent" + schemaVersion string = "v1" +) + +const ( + healthLogsTag string = "ops-agent-health" + agentVersionKey string = "agent.googleapis.com/health/agentVersion" + agentKindKey string = "agent.googleapis.com/health/agentKind" + schemaVersionKey string = "agent.googleapis.com/health/schemaVersion" +) + +// AgentSelfMetrics provides the agent.googleapis.com/agent/ metric and self logs. // It is never referenced in the config file, and instead is forcibly added in confgenerator.go. // Therefore, it does not need to implement any interfaces. type AgentSelfMetrics struct { @@ -34,6 +47,7 @@ type AgentSelfMetrics struct { OtelPort int OtelRuntimeDir string OtlpExporterEnabled bool + LogsDir string } // Following reference : https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto @@ -112,6 +126,18 @@ func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]o Type: "metrics", ReceiverPipelineName: "ops_agent", } + + receiverPipelines["logging_ping"] = r.LoggingPingPipeline(ctx) + pipelines["loggingping"] = otel.Pipeline{ + Type: "logs", + ReceiverPipelineName: "logging_ping", + } + + receiverPipelines["health_checks"] = r.HealthChecksPipeline(ctx) + pipelines["healthchecks"] = otel.Pipeline{ + Type: "logs", + ReceiverPipelineName: "health_checks", + } } func (r AgentSelfMetrics) PrometheusMetricsPipeline(ctx context.Context) otel.ReceiverPipeline { @@ -439,4 +465,136 @@ func (r AgentSelfMetrics) OpsAgentPipeline(ctx context.Context) otel.ReceiverPip }, ctx) } +// This method creates a component that enforces the `Structured Health Logs` format to +// all `ops-agent-health` logs. It sets `agentKind`, `agentVersion` and `schemaVersion`. +// This method also processes all self logs to set the severity field correctly. +func generateStructuredHealthLogsOtelComponents(ctx context.Context) []otel.Component { + components, err := LoggingProcessorModifyFields{ + Fields: map[string]*ModifyField{ + fmt.Sprintf(`labels."%s"`, agentKindKey): { + StaticValue: &agentKind, + }, + fmt.Sprintf(`labels."%s"`, agentVersionKey): { + StaticValue: &version.Version, + }, + fmt.Sprintf(`labels."%s"`, schemaVersionKey): { + StaticValue: &schemaVersion, + }, + "severity": { + MoveFrom: "jsonPayload.severity", + MapValues: map[string]string{ + "error": "ERROR", + "warn": "WARNING", + "info": "INFO", + "debug": "DEBUG", + }, + MapValuesExclusive: false, + }, + }, + }.Processors(ctx) + if err != nil { + // We're generating a hard-coded config, so this should never fail. + panic(err) + } + return components +} + +// generateHealthLogsParsingComponents creates OTel processors that parse health check logs from JSON +// and filter out any lines that do not have a valid severity. +func generateHealthLogsParsingComponents(ctx context.Context) []otel.Component { + components := []otel.Component{} + parseJsonProccesor, err := LoggingProcessorParseJson{ + ParserShared: ParserShared{ + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S%z", + }, + }.Processors(ctx) + if err != nil { + // We're generating a hard-coded config, so this should never fail. + panic(err) + } + components = append(components, parseJsonProccesor...) + + // This is used to exclude any previous content of the `health-checks.log` file that does not contain + // the `jsonPayload.severity` field. + body := ottl.LValue{"body"} + bodySeverity := ottl.LValue{"body", "severity"} + excludeLogFilter := otel.Filter("logs", "log_record", + []ottl.Value{ + ottl.IsNil(body), + ottl.And(body.IsPresent(), ottl.IsNil(bodySeverity)), + ottl.And(bodySeverity.IsPresent(), ottl.Not(ottl.IsMatch(bodySeverity, "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))), + }) + components = append(components, excludeLogFilter) + + return components +} + +func (r AgentSelfMetrics) LoggingPingPipeline(ctx context.Context) otel.ReceiverPipeline { + logProccesors := []otel.Component{ + otel.Transform("log", "log", + []ottl.Statement{ + "set(observed_time, Now())", + "set(time, Now())", + }, + ), + } + logProccesors = append(logProccesors, generateStructuredHealthLogsOtelComponents(ctx)...) + logProccesors = append(logProccesors, otelSetLogNameComponents(ctx, healthLogsTag)...) + + return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{ + Receiver: otel.Component{ + Type: "otlpjsonfile", + Config: map[string]any{ + "include": []string{ + filepath.Join(r.OtelRuntimeDir, "logging_ping_otlp.json"), + }, + "replay_file": true, + "poll_interval": time.Duration(600 * time.Second).String(), + "start_at": "beginning", + }, + }, + Processors: map[string][]otel.Component{ + "logs": logProccesors, + }, + ExporterTypes: map[string]otel.ExporterType{ + "logs": otel.Logging, + }, + }, ctx) +} + +func (r AgentSelfMetrics) HealthChecksPipeline(ctx context.Context) otel.ReceiverPipeline { + healthChecksPath := filepath.Join(r.LogsDir, "health-checks.log") + + logProccesors := generateHealthLogsParsingComponents(ctx) + logProccesors = append(logProccesors, generateStructuredHealthLogsOtelComponents(ctx)...) + logProccesors = append(logProccesors, otelSetLogNameComponents(ctx, healthLogsTag)...) + + return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{ + Receiver: otel.Component{ + Type: "file_log", + Config: map[string]any{ + "include": []string{healthChecksPath}, + "start_at": "beginning", + "storage": fileStorageExtensionType, + "operators": []map[string]any{ + { + "id": "body", + "type": "move", + "from": "body", + "to": "body.message", + }, + }, + }, + }, + Processors: map[string][]otel.Component{ + "logs": logProccesors, + }, + ExporterTypes: map[string]otel.ExporterType{ + "logs": otel.Logging, + }, + UsedExtensions: []string{fileStorageExtensionType}, + }, ctx) +} + // intentionally not registered as a component because this is not created by users diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index 3684735207..10500db2a9 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -241,7 +241,7 @@ func fileStorageExtension(stateDir string) otel.Component { } } -func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDir string) (string, error) { +func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDir, logsDir string) (string, error) { p := platform.FromContext(ctx) userAgent, _ := p.UserAgent("Google-Cloud-Ops-Agent-Metrics") @@ -260,6 +260,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi OtelPort: int(uc.GetOtelMetricsPort()), OtelRuntimeDir: outDir, OtlpExporterEnabled: uc.Global.GetOtlpExporter(), + LogsDir: logsDir, } agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx) diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go index 0c1508e2a5..3e9c2b8001 100644 --- a/confgenerator/confgenerator_test.go +++ b/confgenerator/confgenerator_test.go @@ -281,7 +281,7 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string, } // Otel configs - otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "", "") + otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "", "", "") if err != nil { return } @@ -333,6 +333,12 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string, } got["enabled_receivers_otlp.json"] = string(generatedEnabledReceiversOTLPJSON) + generatedLoggingPingOTLPJSON, err := self_metrics.CollectLoggingPingToOTLPJSON() + if err != nil { + return + } + got["logging_ping_otlp.json"] = string(generatedLoggingPingOTLPJSON) + // If the confgenerator test is designed to test the otel_logging experiment, generate an OTEL config with both otlp_exporter and otel_logging enabled. if len(enabledExperiments) == 1 && enabledExperiments["otel_logging"] { generateOtelConfigWithOtlpExporterEnabled(got, pc, testDir, otelGeneratedConfig) @@ -358,7 +364,7 @@ func generateOtelConfigWithOtlpExporterEnabled(got map[string]string, pc platfor enabled := true mergedUcOtlp.Global.OtlpExporter = &enabled - otelGeneratedConfigOtlp, err := mergedUcOtlp.GenerateOtelConfig(ctxOtlp, "", "") + otelGeneratedConfigOtlp, err := mergedUcOtlp.GenerateOtelConfig(ctxOtlp, "", "", "") if err == nil { got["otel_otlp_exporter.yaml"] = otelGeneratedConfigOtlp } diff --git a/confgenerator/config.go b/confgenerator/config.go index 9ac1665877..26a4f3ace3 100644 --- a/confgenerator/config.go +++ b/confgenerator/config.go @@ -1239,7 +1239,7 @@ func (uc *UnifiedConfig) OTelLoggingSupported(ctx context.Context) bool { } t := true ucLoggingCopy.Logging.Service.OTelLogging = &t - _, err = ucLoggingCopy.GenerateOtelConfig(ctx, "", "") + _, err = ucLoggingCopy.GenerateOtelConfig(ctx, "", "", "") return err == nil } diff --git a/confgenerator/files.go b/confgenerator/files.go index 23afc6b715..0e90c8ba40 100644 --- a/confgenerator/files.go +++ b/confgenerator/files.go @@ -59,7 +59,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l } } case "otel": - otelConfig, err := uc.GenerateOtelConfig(ctx, outDir, stateDir) + otelConfig, err := uc.GenerateOtelConfig(ctx, outDir, stateDir, logsDir) if err != nil { return fmt.Errorf("can't parse configuration: %w", err) } diff --git a/confgenerator/otel/ottl/ottl.go b/confgenerator/otel/ottl/ottl.go index 6de6106e78..89547f9d78 100644 --- a/confgenerator/otel/ottl/ottl.go +++ b/confgenerator/otel/ottl/ottl.go @@ -247,6 +247,10 @@ func Or(conditions ...Value) Value { return valuef(`(%s)`, strings.Join(out, " or ")) } +func IsNil(a Value) Value { + return valuef(`%s == nil`, a) +} + func IsNotNil(a Value) Value { return valuef(`%s != nil`, a) } diff --git a/confgenerator/self_logs.go b/confgenerator/self_logs.go index c678fe3ce1..6c59e5bd6b 100644 --- a/confgenerator/self_logs.go +++ b/confgenerator/self_logs.go @@ -22,23 +22,13 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/internal/healthchecks" - "github.com/GoogleCloudPlatform/ops-agent/internal/logs" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" "github.com/GoogleCloudPlatform/ops-agent/internal/version" ) -var ( - agentKind string = "ops-agent" - schemaVersion string = "v1" -) - const ( opsAgentLogsMatch string = "ops-agent-*" fluentBitSelfLogsTag string = "ops-agent-fluent-bit" - healthLogsTag string = "ops-agent-health" - agentVersionKey string = "agent.googleapis.com/health/agentVersion" - agentKindKey string = "agent.googleapis.com/health/agentKind" - schemaVersionKey string = "agent.googleapis.com/health/schemaVersion" ) func fluentbitSelfLogsPath(p platform.Platform) string { @@ -49,57 +39,6 @@ func fluentbitSelfLogsPath(p platform.Platform) string { return path.Join("${logs_dir}", "subagents", loggingModule) } -func healthChecksLogsPath() string { - return path.Join("${logs_dir}", "health-checks.log") -} - -func generateInputHealthLoggingPingComponent(ctx context.Context) []fluentbit.Component { - return []fluentbit.Component{ - { - Kind: "INPUT", - Config: map[string]string{ - "Name": "dummy", - "Tag": healthLogsTag, - "Dummy": `{"code": "LogPingOpsAgent", "severity": "DEBUG"}`, - "Interval_Sec": "600", - "Interval_NSec": "0", - }, - }, - } -} - -// This method creates a file input for the `health-checks.log` file, a json parser for the -// structured logs and a grep filter to avoid ingesting previous content of the file. -func generateInputHealthChecksLogsComponents(ctx context.Context) []fluentbit.Component { - out := make([]fluentbit.Component, 0) - out = append(out, LoggingReceiverFilesMixin{ - IncludePaths: []string{healthChecksLogsPath()}, - BufferInMemory: true, - }.Components(ctx, healthLogsTag)...) - out = append(out, LoggingProcessorParseJson{ - // TODO(b/282754149): Remove TimeKey and TimeFormat when feature gets implemented. - ParserShared: ParserShared{ - TimeKey: logs.TimeZapKey, - TimeFormat: "%Y-%m-%dT%H:%M:%S%z", - }, - }.Components(ctx, healthLogsTag, "health-checks-json")...) - out = append(out, []fluentbit.Component{ - // This is used to exclude any previous content of the `health-checks.log` file that does not contain - // the `jsonPayload.severity` field. Due to `https://github.com/fluent/fluent-bit/issues/7092` the - // filtering can't be done directly to the `logging.googleapis.com/severity` field. - // We cannot use `LoggingProcessorExcludeLogs` here since it doesn't exclude when the field is missing. - { - Kind: "FILTER", - Config: map[string]string{ - "Name": "grep", - "Match": healthLogsTag, - "Regex": fmt.Sprintf("%s INFO|ERROR|WARNING|DEBUG|info|error|warning|debug", logs.SeverityZapKey), - }, - }, - }...) - return out -} - // This method creates a file input for the `logging-module.log` file, a regex parser for the // fluent-bit self logs and a translator of severity to the logging api format. func generateInputFluentBitSelfLogsComponents(ctx context.Context, logLevel string) []fluentbit.Component { @@ -216,9 +155,7 @@ func generateOutputSelfLogsComponent(ctx context.Context, userAgent string, inge func (uc *UnifiedConfig) generateSelfLogsComponents(ctx context.Context, userAgent string) []fluentbit.Component { out := make([]fluentbit.Component, 0) - out = append(out, generateInputHealthLoggingPingComponent(ctx)...) out = append(out, generateInputFluentBitSelfLogsComponents(ctx, uc.Logging.Service.LogLevel)...) - out = append(out, generateInputHealthChecksLogsComponents(ctx)...) out = append(out, generateFilterSelfLogsSamplingComponents(ctx)...) out = append(out, generateFilterStructuredHealthLogsComponents(ctx)...) out = append(out, generateFilterMapSeverityFieldComponent(ctx)...) diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml index 32661c8a50..d05ab47b51 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -633,6 +640,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -696,6 +883,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -736,6 +933,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -761,6 +964,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/otel.yaml b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/otel.yaml index 44b8d6029f..8a392f5b4d 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -609,6 +616,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -672,6 +859,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -710,6 +907,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -735,6 +938,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml index 72b47b7c87..6443da4602 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -679,6 +866,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -720,6 +917,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -745,6 +948,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml index 72b47b7c87..6443da4602 100644 --- a/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -679,6 +866,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -720,6 +917,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -745,6 +948,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/otel.yaml index ea4a278c6f..d1b6d23cb7 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/otel.yaml index b53080e38d..ab08328624 100644 --- a/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/otel.yaml index a4faaa818a..d3f3217c63 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_journald_0: error_mode: ignore log_statements: @@ -708,6 +832,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "journald") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -763,6 +950,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -791,6 +988,12 @@ receivers: storage: file_storage nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_journald: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/otel.yaml b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/otel.yaml index 1f9dc851ab..cf2ed1ba0b 100644 --- a/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_journald_0: error_mode: ignore log_statements: @@ -679,6 +803,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "journald") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -734,6 +921,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -760,6 +957,12 @@ receivers: priority: debug start_at: beginning storage: file_storage + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_journald: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/otel.yaml b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/otel.yaml b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf index d1bff564db..cc6e571fa2 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match * Name lua @@ -80,30 +58,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/logging_ping_otlp.json b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/otel.yaml b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/otel.yaml index bd32637505..567be573b0 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/otel.yaml +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -754,6 +957,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/builtin/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/builtin/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/builtin/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/builtin/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/builtin/golden/linux/otel.yaml b/confgenerator/testdata/goldens/builtin/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/builtin/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/builtin/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/builtin/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/builtin/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/builtin/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/builtin/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/builtin/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/builtin/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/builtin/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/builtin/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/builtin/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/builtin/golden/windows/otel.yaml b/confgenerator/testdata/goldens/builtin/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/builtin/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/builtin/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml index b434749fcd..e4a4f28b37 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -668,6 +675,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -770,6 +957,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -814,6 +1011,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -839,6 +1042,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml index 2ac846250a..ed09af8bf0 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -639,6 +646,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -741,6 +928,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -783,6 +980,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -808,6 +1011,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml index b500c6e827..cdd23455d4 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1161,6 +1348,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1190,6 +1387,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1271,6 +1474,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml index b500c6e827..cdd23455d4 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1161,6 +1348,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1190,6 +1387,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1271,6 +1474,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/otel.yaml index c498139619..9224a9257a 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -673,6 +680,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -736,6 +923,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -780,6 +977,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -805,6 +1008,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/otel.yaml index ce6ea4e2ed..f970455f06 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -644,6 +651,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -707,6 +894,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -749,6 +946,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -774,6 +977,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/otel.yaml index 213c586192..de7be868e6 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -708,6 +715,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -721,6 +845,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1127,6 +1314,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/otel.yaml index 213c586192..de7be868e6 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -708,6 +715,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -721,6 +845,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1127,6 +1314,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/otel.yaml index 6d9bce0dd0..9354081aa1 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/otel.yaml @@ -67,6 +67,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -744,6 +931,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -813,6 +1016,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/otel.yaml index 1c7ec8c72a..4d0b4961fe 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -651,6 +658,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -714,6 +901,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -756,6 +953,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -781,6 +984,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/otel.yaml index c5a4274c56..750ac5c498 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/otel.yaml @@ -73,6 +73,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -730,6 +854,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1136,6 +1323,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/otel.yaml index c5a4274c56..750ac5c498 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/otel.yaml @@ -73,6 +73,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -730,6 +854,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1136,6 +1323,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/otel.yaml index 059eb0cb64..bf05ded5ca 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -668,6 +675,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -741,6 +928,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -785,6 +982,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -810,6 +1013,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/otel.yaml index 59c0080de1..678baf1d0b 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -639,6 +646,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -712,6 +899,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -754,6 +951,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -779,6 +982,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/otel.yaml index d1459b8b83..2971da42b0 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/otel.yaml index d1459b8b83..2971da42b0 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/otel.yaml index 059eb0cb64..bf05ded5ca 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -668,6 +675,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -741,6 +928,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -785,6 +982,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -810,6 +1013,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/otel.yaml index 59c0080de1..678baf1d0b 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -639,6 +646,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -712,6 +899,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -754,6 +951,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -779,6 +982,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/otel.yaml index d1459b8b83..2971da42b0 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/otel.yaml index d1459b8b83..2971da42b0 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/otel.yaml index 585f2ee41c..689ec5bc15 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -668,6 +675,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -741,6 +928,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -785,6 +982,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -810,6 +1013,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/otel.yaml index 1db0a90a91..1a5c298a1b 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -639,6 +646,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -712,6 +899,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -754,6 +951,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -779,6 +982,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/otel.yaml index a767294a08..62f4a4f4b1 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/otel.yaml index a767294a08..62f4a4f4b1 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_multiple_pipelines/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -716,6 +840,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1132,6 +1319,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1161,6 +1358,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1242,6 +1445,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/otel.yaml index cf0e509678..db7ac25c38 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -729,6 +916,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -773,6 +970,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -798,6 +1001,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/otel.yaml index e7f9029cc2..d90f8c02d9 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -700,6 +887,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -742,6 +939,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -767,6 +970,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/otel.yaml index cb63924b4f..e96a8eaebb 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1120,6 +1307,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1149,6 +1346,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1230,6 +1433,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/otel.yaml index cb63924b4f..e96a8eaebb 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_no_traces/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1120,6 +1307,16 @@ processors: - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1149,6 +1346,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1230,6 +1433,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml index eb84f88868..1539d5f754 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux-gpu/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -906,6 +913,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -965,6 +1089,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "file_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_file__logs: exclude: [] fingerprint_size: 5kb @@ -1216,6 +1413,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml index 213d3a8383..0454ad5092 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/linux/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -877,6 +884,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -921,6 +1045,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "file_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1109,6 +1296,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_file__logs: exclude: [] fingerprint_size: 5kb @@ -1170,6 +1367,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1200,6 +1403,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml index 67a58984ad..b69d3030de 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows-2012/otel.yaml @@ -79,6 +79,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -941,6 +948,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1013,6 +1137,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "file_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog__custom_0: error_mode: ignore log_statements: @@ -1559,6 +1746,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_file__logs: exclude: [] fingerprint_size: 5kb @@ -1607,6 +1804,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1693,6 +1896,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml index 67a58984ad..b69d3030de 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otel_logging_otlpgrpc_exporter/golden/windows/otel.yaml @@ -79,6 +79,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -941,6 +948,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1013,6 +1137,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "file_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog__custom_0: error_mode: ignore log_statements: @@ -1559,6 +1746,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_file__logs: exclude: [] fingerprint_size: 5kb @@ -1607,6 +1804,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1693,6 +1896,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml index 44482a95d0..2f04963143 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux-gpu/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -906,6 +913,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -936,6 +1060,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1082,6 +1269,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1130,6 +1327,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1156,6 +1359,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml index cc069f533e..72cc26bf0c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/linux/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -877,6 +884,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -892,6 +1016,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1038,6 +1225,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1084,6 +1281,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1110,6 +1313,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml index ec559da98a..dff8eb2477 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows-2012/otel.yaml @@ -79,6 +79,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -941,6 +948,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -984,6 +1108,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1488,6 +1675,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1521,6 +1718,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1603,6 +1806,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml index ec559da98a..dff8eb2477 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp_otlpgrpc_exporter/golden/windows/otel.yaml @@ -79,6 +79,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -941,6 +948,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -984,6 +1108,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1488,6 +1675,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1521,6 +1718,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1603,6 +1806,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/otel.yaml index c3679e50a5..7f2066cab8 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -666,6 +673,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -864,6 +1051,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -923,6 +1120,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -955,6 +1158,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/otel.yaml index 8cd1cec6de..d6c74ebb83 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -637,6 +644,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -835,6 +1022,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -892,6 +1089,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -924,6 +1127,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/otel.yaml index f30833cbc9..4fa08b55d4 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -1255,6 +1442,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -1299,6 +1496,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1387,6 +1590,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/otel.yaml index f30833cbc9..4fa08b55d4 100644 --- a/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_redis/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -1255,6 +1442,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -1299,6 +1496,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1387,6 +1590,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/global-default-log-rotation_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_main.conf index c8c034cb63..d165e6bcee 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_main.conf index c8c034cb63..d165e6bcee 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_main.conf index f869006ec4..9c639ea5a2 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_main.conf index f869006ec4..9c639ea5a2 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_false/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/otel.yaml b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/global-service_default_self_log_file_collection_true/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_main.conf index 38c8c523ec..ea5638db2f 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_main.conf index 38c8c523ec..ea5638db2f 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_main.conf index ee05da029c..860dc65839 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_main.conf index ee05da029c..860dc65839 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-custom_log_level/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf index a41f7f5311..5bd44fae26 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf @@ -32,13 +32,6 @@ Tag default_pipeline.syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -54,21 +47,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match default_pipeline.syslog Name lua @@ -95,30 +73,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/otel.yaml index 5c4b768471..37df5c8671 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -39,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -625,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -659,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -683,6 +898,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -705,7 +926,30 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_main.conf index a41f7f5311..5bd44fae26 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_main.conf @@ -32,13 +32,6 @@ Tag default_pipeline.syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -54,21 +47,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match default_pipeline.syslog Name lua @@ -95,30 +73,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/otel.yaml index 56ed3b6739..67446d1022 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -34,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -596,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -630,6 +835,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -652,6 +867,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -674,7 +895,30 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_main.conf index 15d4d8eb89..ef4d68247d 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_main.conf @@ -25,13 +25,6 @@ String_Inserts true Tag default_pipeline.windows_event_log -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -47,21 +40,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match default_pipeline.windows_event_log Name lua @@ -138,30 +116,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_parser.conf index 42d5c07fac..2a5542374b 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/fluent_bit_parser.conf @@ -12,9 +12,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/otel.yaml index 71cc037d5f..349aff1be7 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows-2012/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -44,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -660,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -673,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -713,6 +918,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -738,6 +953,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -792,7 +1013,30 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_main.conf index 15d4d8eb89..ef4d68247d 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_main.conf @@ -25,13 +25,6 @@ String_Inserts true Tag default_pipeline.windows_event_log -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -47,21 +40,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match default_pipeline.windows_event_log Name lua @@ -138,30 +116,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_parser.conf index 42d5c07fac..2a5542374b 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/fluent_bit_parser.conf @@ -12,9 +12,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/otel.yaml index 71cc037d5f..349aff1be7 100644 --- a/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/windows/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -44,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -660,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -673,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -713,6 +918,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -738,6 +953,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -792,7 +1013,30 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/otel.yaml index 5c4b768471..37df5c8671 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux-gpu/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -39,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -625,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -659,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -683,6 +898,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -705,7 +926,30 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/otel.yaml index 56ed3b6739..67446d1022 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/linux/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -34,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -596,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -630,6 +835,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -652,6 +867,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -674,7 +895,30 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/otel.yaml index 71cc037d5f..349aff1be7 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows-2012/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -44,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -660,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -673,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -713,6 +918,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -738,6 +953,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -792,7 +1013,30 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/otel.yaml index 71cc037d5f..349aff1be7 100644 --- a/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-default_overrides_disable_all/golden/windows/otel.yaml @@ -7,6 +7,24 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logging: + sending_queue: + batch: + flush_timeout: 200ms + max_size: 1000 + min_size: 1000 + sizer: items + block_on_overflow: true + enabled: true + num_consumers: 10 + queue_size: 12000 + sizer: items + storage: file_storage + timeout: 3600s +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: @@ -44,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -660,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -673,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/loggingmetrics_0: error_mode: ignore metric_statements: @@ -713,6 +918,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -738,6 +953,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -792,7 +1013,30 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping metrics/default__pipeline_hostmetrics: exporters: - googlecloud diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml index 9ce0f9c536..a39f2963d1 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -759,6 +766,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -851,6 +1038,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -906,6 +1103,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -931,6 +1134,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel_otlp_exporter.yaml index e60fc4662f..a35ac328d4 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -976,6 +983,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -1006,6 +1130,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1183,6 +1380,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1209,6 +1412,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml index f2beb9edb4..69cf89cb10 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -730,6 +737,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -822,6 +1009,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -875,6 +1072,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -900,6 +1103,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel_otlp_exporter.yaml index 4a7c332e9c..ea0e76f2fd 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -947,6 +954,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -962,6 +1086,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1084,6 +1271,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1137,6 +1334,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1163,6 +1366,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml index 769bcbd7c6..30f8b46c9b 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -794,6 +801,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1282,6 +1479,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1363,6 +1566,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel_otlp_exporter.yaml index b6c032280a..5853346b63 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -1011,6 +1018,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1054,6 +1178,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1534,6 +1721,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1574,6 +1771,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1656,6 +1859,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml index 769bcbd7c6..30f8b46c9b 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -794,6 +801,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1282,6 +1479,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1363,6 +1566,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel_otlp_exporter.yaml index b6c032280a..5853346b63 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -1011,6 +1018,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1054,6 +1178,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1534,6 +1721,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1574,6 +1771,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1656,6 +1859,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml index 48d497cbef..4c48e7dab6 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -824,6 +1011,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -879,6 +1076,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -904,6 +1107,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel_otlp_exporter.yaml index 873e554fc9..f5416371fa 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1101,6 +1288,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1156,6 +1353,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1182,6 +1385,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml index 08ccecfd2e..7ed88cbf9a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -795,6 +982,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -848,6 +1045,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -873,6 +1076,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel_otlp_exporter.yaml index 4fd6c6fbeb..6886507300 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1057,6 +1244,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1110,6 +1307,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1136,6 +1339,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml index fc0be1ca87..ee99a0f01d 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1215,6 +1402,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1255,6 +1452,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1336,6 +1539,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel_otlp_exporter.yaml index a2e9ae768b..85c334514a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1507,6 +1694,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1547,6 +1744,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1629,6 +1832,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml index fc0be1ca87..ee99a0f01d 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1215,6 +1402,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1255,6 +1452,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1336,6 +1539,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel_otlp_exporter.yaml index a2e9ae768b..85c334514a 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1507,6 +1694,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1547,6 +1744,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1629,6 +1832,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml index 5b1beb9cc3..f070db9713 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -750,6 +937,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -794,6 +991,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -819,6 +1022,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml index 2a9d327a2f..939b4146c9 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -721,6 +908,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -763,6 +960,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -788,6 +991,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml index f0bd675a3d..ebc3f7a090 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1141,6 +1328,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1170,6 +1367,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml index f0bd675a3d..ebc3f7a090 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1141,6 +1328,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1170,6 +1367,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml index 8bcbf606e5..b8b87c69ec 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -796,6 +983,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -852,6 +1049,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -877,6 +1080,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel_otlp_exporter.yaml index 52ee9e7506..ebbfd1971f 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -919,6 +1043,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1073,6 +1260,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1129,6 +1326,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1155,6 +1358,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml index 11d7c62526..5451edabac 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -643,6 +767,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -767,6 +954,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -821,6 +1018,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -846,6 +1049,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel_otlp_exporter.yaml index 51552b18f5..2ff7140079 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -875,6 +999,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1029,6 +1216,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1083,6 +1280,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1109,6 +1312,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml index 486136d0f8..17d6428c39 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1187,6 +1374,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1228,6 +1425,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1309,6 +1512,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel_otlp_exporter.yaml index 371b7f5b18..bcff7e3225 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -967,6 +1091,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1479,6 +1666,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1520,6 +1717,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1602,6 +1805,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml index 486136d0f8..17d6428c39 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1187,6 +1374,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1228,6 +1425,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1309,6 +1512,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel_otlp_exporter.yaml index 371b7f5b18..bcff7e3225 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -967,6 +1091,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1479,6 +1666,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1520,6 +1717,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1602,6 +1805,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml index 6a1f70d179..ceed44aa14 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -767,6 +954,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -807,6 +1004,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -832,6 +1035,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel_otlp_exporter.yaml index 93d4d6a91d..12383336a4 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1044,6 +1231,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1084,6 +1281,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1110,6 +1313,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml index a21a75c58e..7998fc35d6 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -738,6 +925,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -776,6 +973,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -801,6 +1004,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel_otlp_exporter.yaml index 79ebb87475..0ad530b2c9 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1000,6 +1187,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1038,6 +1235,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1064,6 +1267,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml index fd74ca1704..15cf98f284 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1280,6 +1467,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1305,6 +1502,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1386,6 +1589,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel_otlp_exporter.yaml index 7dbfa9d677..b01c5062c0 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1572,6 +1759,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1597,6 +1794,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1679,6 +1882,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml index fd74ca1704..15cf98f284 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1280,6 +1467,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1305,6 +1502,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1386,6 +1589,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel_otlp_exporter.yaml index 7dbfa9d677..b01c5062c0 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1572,6 +1759,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1597,6 +1794,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1679,6 +1882,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml index 3b7214ed6f..13ac359aa7 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -735,6 +922,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel_otlp_exporter.yaml index 9dd88a9616..ae0212c270 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -919,6 +1043,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1012,6 +1199,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1068,6 +1265,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1094,6 +1297,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml index c059551468..f268c49251 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -643,6 +767,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel_otlp_exporter.yaml index 0e1d440aa2..9efbd2202e 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -875,6 +999,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -968,6 +1155,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1022,6 +1219,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1048,6 +1251,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml index 5e112ead07..b5283396cb 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel_otlp_exporter.yaml index 888d0240a6..182c4411a9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -967,6 +1091,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1418,6 +1605,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1459,6 +1656,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1541,6 +1744,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml index 5e112ead07..b5283396cb 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel_otlp_exporter.yaml index 888d0240a6..182c4411a9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -967,6 +1091,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1418,6 +1605,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1459,6 +1656,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1541,6 +1744,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel.yaml index 9cea5c94aa..32e7a96784 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward_0: error_mode: ignore log_statements: @@ -689,6 +813,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -752,6 +939,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -794,6 +991,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -819,6 +1022,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel_otlp_exporter.yaml index b5c3451913..f2b483b7a2 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -936,6 +1060,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1029,6 +1216,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1071,6 +1268,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1097,6 +1300,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel.yaml index 0a6f3d9580..043a916413 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward_0: error_mode: ignore log_statements: @@ -660,6 +784,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -723,6 +910,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -763,6 +960,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -788,6 +991,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel_otlp_exporter.yaml index aac860b7eb..9b095bc1fe 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -892,6 +1016,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -985,6 +1172,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1025,6 +1222,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1051,6 +1254,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel.yaml index 5c818daba0..a374e090cd 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel_otlp_exporter.yaml index 4a4a6657e7..82055dbafa 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -984,6 +1108,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1435,6 +1622,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1462,6 +1659,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1544,6 +1747,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel.yaml index 5c818daba0..a374e090cd 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel_otlp_exporter.yaml index 4a4a6657e7..82055dbafa 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_forward/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -984,6 +1108,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1435,6 +1622,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1462,6 +1659,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1544,6 +1747,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml index 35d305ff70..1ae36645eb 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka_1: error_mode: ignore log_statements: @@ -767,6 +891,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -830,6 +1017,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -887,6 +1084,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -912,6 +1115,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel_otlp_exporter.yaml index 1b7ff4f9dc..fbb04b7e6e 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -873,6 +880,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -1014,6 +1138,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1107,6 +1294,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1164,6 +1361,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1190,6 +1393,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml index b3cf901630..74f931c887 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka_1: error_mode: ignore log_statements: @@ -738,6 +862,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -856,6 +1053,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -881,6 +1084,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel_otlp_exporter.yaml index 548f6d7b37..e0acc973f7 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -844,6 +851,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -970,6 +1094,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1063,6 +1250,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1118,6 +1315,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1144,6 +1347,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml index 030f8f4ab0..5ade14113a 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel_otlp_exporter.yaml index 02901d5d32..5c48339c2f 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -908,6 +915,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1062,6 +1186,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1513,6 +1700,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1555,6 +1752,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1637,6 +1840,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml index 030f8f4ab0..5ade14113a 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel_otlp_exporter.yaml index 02901d5d32..5c48339c2f 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -908,6 +915,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1062,6 +1186,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1513,6 +1700,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1555,6 +1752,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1637,6 +1840,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel.yaml index 12a24e69f6..d3ac10af79 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mongodb_0: error_mode: ignore log_statements: @@ -985,6 +1109,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1048,6 +1235,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1103,6 +1300,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1128,6 +1331,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel_otlp_exporter.yaml index 5dbe236a12..fb3f69e8d6 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -1232,6 +1356,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1325,6 +1512,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1380,6 +1577,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1406,6 +1609,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel.yaml index b2d291bb13..987b3bd0e5 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mongodb_0: error_mode: ignore log_statements: @@ -956,6 +1080,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1019,6 +1206,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1072,6 +1269,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1097,6 +1300,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel_otlp_exporter.yaml index d857e2e4e0..178a618445 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1188,6 +1312,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1281,6 +1468,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1334,6 +1531,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1360,6 +1563,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel.yaml index 02b2cfba75..63e4f9e3fe 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1033,6 +1157,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1439,6 +1626,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1479,6 +1676,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1560,6 +1763,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel_otlp_exporter.yaml index bf2b384b95..731d9631e7 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1280,6 +1404,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1731,6 +1918,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1771,6 +1968,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1853,6 +2056,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel.yaml index 02b2cfba75..63e4f9e3fe 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1033,6 +1157,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1439,6 +1626,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1479,6 +1676,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1560,6 +1763,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel_otlp_exporter.yaml index bf2b384b95..731d9631e7 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mongodb/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1280,6 +1404,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1731,6 +1918,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1771,6 +1968,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1853,6 +2056,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml index 91ba188e67..8ac0f0eb00 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__access_0: error_mode: ignore log_statements: @@ -933,6 +1057,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -996,6 +1183,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1066,6 +1263,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1091,6 +1294,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel_otlp_exporter.yaml index 601784d2fd..d1ff4a8325 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -1180,6 +1304,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1273,6 +1460,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1343,6 +1540,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1369,6 +1572,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml index 196b775c97..b98c4af05f 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__access_0: error_mode: ignore log_statements: @@ -904,6 +1028,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -967,6 +1154,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1035,6 +1232,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1060,6 +1263,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel_otlp_exporter.yaml index 807f7c820b..c9df630f1c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1136,6 +1260,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1229,6 +1416,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1297,6 +1494,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1323,6 +1526,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml index 08bde18bc5..35dac314b9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -981,6 +1105,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1387,6 +1574,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1442,6 +1639,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1523,6 +1726,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel_otlp_exporter.yaml index d4ffca3016..7e8f5b194d 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1228,6 +1352,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1679,6 +1866,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1734,6 +1931,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1816,6 +2019,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml index 08bde18bc5..35dac314b9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -981,6 +1105,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1387,6 +1574,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1442,6 +1639,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1523,6 +1726,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel_otlp_exporter.yaml index d4ffca3016..7e8f5b194d 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -1228,6 +1352,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1679,6 +1866,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1734,6 +1931,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1816,6 +2019,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml index 40ef488271..45c9517ed9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -883,6 +1070,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -907,6 +1104,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -940,6 +1143,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel_otlp_exporter.yaml index aca460ab00..a3bb6c56d6 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1160,6 +1347,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1184,6 +1381,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1218,6 +1421,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml index f27d9c3a69..ce1ffb7cde 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -854,6 +1041,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -876,6 +1073,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -909,6 +1112,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel_otlp_exporter.yaml index 26e2c6424c..c7a6093e3b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1116,6 +1303,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1138,6 +1335,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1172,6 +1375,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml index 27b5d5c5db..3a5d630e14 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -937,6 +1124,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -962,6 +1159,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1027,6 +1230,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel_otlp_exporter.yaml index dda1a3a80e..b3fbbc63dc 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1229,6 +1416,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1254,6 +1451,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1320,6 +1523,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml index 27b5d5c5db..3a5d630e14 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -937,6 +1124,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -962,6 +1159,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1027,6 +1230,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel_otlp_exporter.yaml index dda1a3a80e..b3fbbc63dc 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1229,6 +1416,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1254,6 +1451,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1320,6 +1523,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml index 6f6b1693c6..353eb1f650 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -771,6 +958,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -815,6 +1012,12 @@ receivers: storage: file_storage nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -840,6 +1043,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel_otlp_exporter.yaml index 5f2c0b97e2..3b62d7dd82 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1048,6 +1235,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1092,6 +1289,12 @@ receivers: storage: file_storage nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1118,6 +1321,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml index 3f7f987bbe..7674e69dfd 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: priority: debug start_at: beginning storage: file_storage + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel_otlp_exporter.yaml index dbdae9189c..2ff23cccfe 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1004,6 +1191,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1046,6 +1243,12 @@ receivers: priority: debug start_at: beginning storage: file_storage + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1072,6 +1275,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel.yaml index bacca5dcb1..a49e013668 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -667,6 +674,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -785,6 +972,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -829,6 +1026,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -854,6 +1057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel_otlp_exporter.yaml index ae719e912f..4ad59443c4 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux-gpu/otel_otlp_exporter.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -884,6 +891,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -914,6 +1038,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1062,6 +1249,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1106,6 +1303,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1132,6 +1335,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel.yaml index fa074f6cff..9e75ac022d 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -638,6 +645,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -756,6 +943,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -798,6 +995,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -823,6 +1026,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel_otlp_exporter.yaml index b57f019ae8..c65a7d60aa 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/linux/otel_otlp_exporter.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -855,6 +862,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -870,6 +994,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1018,6 +1205,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1060,6 +1257,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1086,6 +1289,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel.yaml index dfd36d820a..7df7623829 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -702,6 +709,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -715,6 +839,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1176,6 +1363,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1205,6 +1402,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1286,6 +1489,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel_otlp_exporter.yaml index c5b89c85d2..a009ea663c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -919,6 +926,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -962,6 +1086,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1468,6 +1655,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1497,6 +1694,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1579,6 +1782,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel.yaml index dfd36d820a..7df7623829 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -702,6 +709,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -715,6 +839,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1176,6 +1363,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1205,6 +1402,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1286,6 +1489,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel_otlp_exporter.yaml index c5b89c85d2..a009ea663c 100644 --- a/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/logging-otel-ruby_regex/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -919,6 +926,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -962,6 +1086,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1468,6 +1655,16 @@ processors: - set(cache["value"], "sample_logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1497,6 +1694,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1579,6 +1782,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/otel.yaml index ed67b35500..3833bcbee8 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -701,6 +825,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -941,6 +1128,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1001,6 +1198,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1034,6 +1237,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/otel.yaml index 0b74831242..ebc72d14d1 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -912,6 +1099,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -970,6 +1167,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1003,6 +1206,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml index 3838d2110b..b5a5f58580 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -749,6 +873,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -995,6 +1182,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1056,6 +1253,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1121,6 +1324,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/otel.yaml index 3838d2110b..b5a5f58580 100644 --- a/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-pipeline_multiple_pipelines/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -749,6 +873,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -995,6 +1182,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1056,6 +1253,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1121,6 +1324,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/otel.yaml index 9ce0f9c536..a39f2963d1 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -759,6 +766,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -851,6 +1038,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -906,6 +1103,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -931,6 +1134,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/otel.yaml index f2beb9edb4..69cf89cb10 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -730,6 +737,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -822,6 +1009,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -875,6 +1072,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -900,6 +1103,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/otel.yaml index 769bcbd7c6..30f8b46c9b 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -794,6 +801,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1282,6 +1479,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1363,6 +1566,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/otel.yaml index 769bcbd7c6..30f8b46c9b 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -794,6 +801,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1282,6 +1479,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1363,6 +1566,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/otel.yaml index 5bb8dbc9b6..b712df21e5 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -648,6 +655,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_logs_0: error_mode: ignore log_statements: @@ -677,6 +801,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_logs: exclude: [] fingerprint_size: 5kb @@ -856,6 +1053,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -881,6 +1084,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/otel.yaml index 3f2240cb82..52dede6ae0 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -619,6 +626,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_logs_0: error_mode: ignore log_statements: @@ -648,6 +772,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -772,6 +959,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_logs: exclude: [] fingerprint_size: 5kb @@ -825,6 +1022,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -850,6 +1053,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/otel.yaml index 15aa314456..82f2f294de 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -725,6 +849,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1192,6 +1379,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_logs: exclude: [] fingerprint_size: 5kb @@ -1232,6 +1429,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1313,6 +1516,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/otel.yaml index 15aa314456..82f2f294de 100644 --- a/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_exclude_logs_contains/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -725,6 +849,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "logs") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1192,6 +1379,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_logs: exclude: [] fingerprint_size: 5kb @@ -1232,6 +1429,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1313,6 +1516,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/otel.yaml index 86fd4ff69b..01592d2d20 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -819,6 +1006,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -874,6 +1071,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -899,6 +1102,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/otel.yaml index 75d567d06f..ce67d814b8 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -790,6 +977,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -843,6 +1040,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -868,6 +1071,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/otel.yaml index 9ae13013a5..84ed432225 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1210,6 +1397,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1250,6 +1447,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1331,6 +1534,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/otel.yaml index 9ae13013a5..84ed432225 100644 --- a/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_modify_fields/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1210,6 +1397,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1250,6 +1447,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1331,6 +1534,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/otel.yaml index 855b073580..92dbe1b08d 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -963,6 +1150,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1018,6 +1215,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1043,6 +1246,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/otel.yaml index fc9ec8dc37..a98590c8b6 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -934,6 +1121,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -987,6 +1184,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1012,6 +1215,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/otel.yaml index 8cf01ac17e..a744fed3dc 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1354,6 +1541,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1394,6 +1591,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1475,6 +1678,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/otel.yaml index 8cf01ac17e..a744fed3dc 100644 --- a/confgenerator/testdata/goldens/logging-processor_order/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_order/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_sample__logs_0: error_mode: ignore log_statements: @@ -1354,6 +1541,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_sample__logs: exclude: [] fingerprint_size: 5kb @@ -1394,6 +1591,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1475,6 +1678,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/otel.yaml index 7b8539d864..81d04c4bfa 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -701,6 +825,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1063,6 +1250,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1123,6 +1320,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1156,6 +1359,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/otel.yaml index df2d1ccf5e..a86e613cbe 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1034,6 +1221,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1092,6 +1289,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1125,6 +1328,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/otel.yaml index 370968d20f..71b4920771 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -749,6 +873,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1117,6 +1304,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/otel.yaml index 370968d20f..71b4920771 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_json_and_parse_regex_types/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -749,6 +873,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id2") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -1117,6 +1304,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: - /path/to/log/1/a/exclude_a @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_log__source__id1: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_parser.conf index 13664dd6b1..2e079ad8ef 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_parser.conf index 13664dd6b1..2e079ad8ef 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_parser.conf index 13664dd6b1..2e079ad8ef 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_parser.conf index 13664dd6b1..2e079ad8ef 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_main.conf index 261b2f3813..23122ab9e3 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_main.conf @@ -39,13 +39,6 @@ Name systemd Tag p1.systemd -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -61,21 +54,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -204,30 +182,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_parser.conf index 3924c8bbb1..377f6cc3cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_main.conf index 261b2f3813..23122ab9e3 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_main.conf @@ -39,13 +39,6 @@ Name systemd Tag p1.systemd -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -61,21 +54,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -204,30 +182,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_parser.conf index 3924c8bbb1..377f6cc3cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_journald_receiver/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_main.conf index 84696bc05a..093c4b0641 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_main.conf @@ -33,13 +33,6 @@ Tag p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name lua @@ -126,30 +104,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_parser.conf index f71b4745b7..83d94cb152 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1.1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_main.conf index 84696bc05a..093c4b0641 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_main.conf @@ -33,13 +33,6 @@ Tag p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name lua @@ -126,30 +104,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_parser.conf index f71b4745b7..83d94cb152 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1.1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_main.conf index 5cfc2fba7a..5314ea0a5e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_main.conf @@ -33,13 +33,6 @@ Tag p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name lua @@ -126,30 +104,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_parser.conf index f71b4745b7..83d94cb152 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1.1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_main.conf index 5cfc2fba7a..5314ea0a5e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_main.conf @@ -33,13 +33,6 @@ Tag p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name lua @@ -126,30 +104,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_parser.conf index f71b4745b7..83d94cb152 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1.1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_not_first/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/otel.yaml index 05ac120059..f29c31b621 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_files__1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "files_1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -735,6 +922,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_files__1: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/otel.yaml index c6cf321009..b6269e5414 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_files__1_0: error_mode: ignore log_statements: @@ -643,6 +767,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "files_1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_files__1: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/otel.yaml index 0195acde2d..3f2e517eb7 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "files_1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_files__1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/otel.yaml index 0195acde2d..3f2e517eb7 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_processor_not_in_use/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "files_1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_files__1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_parser.conf index 25522451e3..7db375676e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_parser.conf index 25522451e3..7db375676e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_parser.conf index 25522451e3..7db375676e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_parser.conf index 25522451e3..7db375676e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_languages/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_main.conf index ead8ab5655..3c64b14eba 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_main.conf @@ -68,13 +68,6 @@ multiline.parser multiline.p3.files_3 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -90,21 +83,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -176,30 +154,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_parser.conf index 8f11bf7b8b..428545d586 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_main.conf index ead8ab5655..3c64b14eba 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_main.conf @@ -68,13 +68,6 @@ multiline.parser multiline.p3.files_3 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -90,21 +83,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -176,30 +154,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_parser.conf index 8f11bf7b8b..428545d586 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_main.conf index 214350c3d1..bc5e3f433f 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_main.conf @@ -68,13 +68,6 @@ multiline.parser multiline.p3.files_3 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -90,21 +83,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -176,30 +154,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_parser.conf index 8f11bf7b8b..428545d586 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_main.conf index 214350c3d1..bc5e3f433f 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_main.conf @@ -68,13 +68,6 @@ multiline.parser multiline.p3.files_3 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -90,21 +83,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -176,30 +154,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_parser.conf index 8f11bf7b8b..428545d586 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_three_processors_same_language/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_parser.conf index e0dcfde663..74a5a0c5b1 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_main.conf index 2b68fa38c2..cdc44189bb 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_parser.conf index e0dcfde663..74a5a0c5b1 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_parser.conf index e0dcfde663..74a5a0c5b1 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_main.conf index 2d8c6e1d81..599b5f09ac 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.p1.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -108,30 +86,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_parser.conf index e0dcfde663..74a5a0c5b1 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_languages/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_main.conf index 64c8e24e1e..dc38953d4c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_main.conf @@ -51,13 +51,6 @@ multiline.parser multiline.p2.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,21 +66,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -142,30 +120,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_parser.conf index ab094c41bb..374b98f120 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_main.conf index 64c8e24e1e..dc38953d4c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_main.conf @@ -51,13 +51,6 @@ multiline.parser multiline.p2.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,21 +66,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -142,30 +120,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_parser.conf index ab094c41bb..374b98f120 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_main.conf index 0139813f3c..16d98b7a17 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_main.conf @@ -51,13 +51,6 @@ multiline.parser multiline.p2.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,21 +66,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -142,30 +120,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_parser.conf index ab094c41bb..374b98f120 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_main.conf index 0139813f3c..16d98b7a17 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_main.conf @@ -51,13 +51,6 @@ multiline.parser multiline.p2.files_1 storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,21 +66,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match p1.files_1 Name modify @@ -142,30 +120,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_parser.conf index ab094c41bb..374b98f120 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/fluent_bit_parser.conf @@ -6,12 +6,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.p1.files_1 diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_multiline_two_processors/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml index 6a1f70d179..ceed44aa14 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -767,6 +954,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -807,6 +1004,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -832,6 +1035,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml index a21a75c58e..7998fc35d6 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -738,6 +925,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -776,6 +973,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -801,6 +1004,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml index fd74ca1704..15cf98f284 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1280,6 +1467,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1305,6 +1502,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1386,6 +1589,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml index fd74ca1704..15cf98f284 100644 --- a/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1280,6 +1467,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1305,6 +1502,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1386,6 +1589,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/otel.yaml index 411929cefb..e960166e21 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_apache__access_0: error_mode: ignore log_statements: @@ -940,6 +1064,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1003,6 +1190,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__access: exclude: [] fingerprint_size: 5kb @@ -1077,6 +1274,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1102,6 +1305,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache_logging_apache__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/otel.yaml index 233ded42ac..e6a7f005c3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_apache__access_0: error_mode: ignore log_statements: @@ -911,6 +1035,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -974,6 +1161,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__access: exclude: [] fingerprint_size: 5kb @@ -1046,6 +1243,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1071,6 +1274,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache_logging_apache__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/otel.yaml index 69fc7f3505..d91148f6d4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -988,6 +1112,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1394,6 +1581,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__access: exclude: [] fingerprint_size: 5kb @@ -1453,6 +1650,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1534,6 +1737,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache_logging_apache__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/otel.yaml index 69fc7f3505..d91148f6d4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -988,6 +1112,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1394,6 +1581,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__access: exclude: [] fingerprint_size: 5kb @@ -1453,6 +1650,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1534,6 +1737,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache_logging_apache__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/otel.yaml index 144f190623..158f3f2191 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_apache__custom__access_0: error_mode: ignore log_statements: @@ -1321,6 +1445,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1623,6 +1810,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__custom__access: exclude: [] fingerprint_size: 5kb @@ -1729,6 +1926,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1762,6 +1965,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache__custom_logging_apache__custom__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/otel.yaml index e4018e221a..65ffab1945 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_apache__custom__access_0: error_mode: ignore log_statements: @@ -1292,6 +1416,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1594,6 +1781,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__custom__access: exclude: [] fingerprint_size: 5kb @@ -1698,6 +1895,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1731,6 +1934,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache__custom_logging_apache__custom__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/otel.yaml index ceefb1a3e9..d9a26dd0a4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1369,6 +1493,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -2014,6 +2201,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__custom__access: exclude: [] fingerprint_size: 5kb @@ -2105,6 +2302,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2194,6 +2397,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache__custom_logging_apache__custom__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/otel.yaml index ceefb1a3e9..d9a26dd0a4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_apache_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1369,6 +1493,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "apache_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -2014,6 +2201,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_apache__custom__access: exclude: [] fingerprint_size: 5kb @@ -2105,6 +2302,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2194,6 +2397,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_apache__custom_logging_apache__custom__access: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/otel.yaml index e3eac611d9..7c80708d85 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_cassandra__debug_1: error_mode: ignore log_statements: @@ -1084,6 +1208,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1147,6 +1334,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__debug: exclude: [] fingerprint_size: 5kb @@ -1233,6 +1430,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1258,6 +1461,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra_logging_cassandra__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/otel.yaml index b8c5b3325f..9832acc518 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -653,6 +660,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_cassandra__debug_1: error_mode: ignore log_statements: @@ -1055,6 +1179,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1118,6 +1305,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__debug: exclude: [] fingerprint_size: 5kb @@ -1202,6 +1399,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1227,6 +1430,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra_logging_cassandra__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/otel.yaml index bd386d8e76..ab06466ed8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1132,6 +1256,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1538,6 +1725,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__debug: exclude: [] fingerprint_size: 5kb @@ -1609,6 +1806,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1690,6 +1893,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra_logging_cassandra__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/otel.yaml index bd386d8e76..ab06466ed8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1132,6 +1256,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1538,6 +1725,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__debug: exclude: [] fingerprint_size: 5kb @@ -1609,6 +1806,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1690,6 +1893,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra_logging_cassandra__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/otel.yaml index 89bec5cb2c..62931c4cea 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -838,6 +845,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_cassandra__custom__debug_1: error_mode: ignore log_statements: @@ -1768,6 +1892,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_syslog_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -2776,6 +2963,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__custom__debug: exclude: [] fingerprint_size: 5kb @@ -2910,6 +3107,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2947,6 +3150,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra__custom_logging_cassandra__custom__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/otel.yaml index 7fc9c1c421..9806bc9975 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -809,6 +816,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_cassandra__custom__debug_1: error_mode: ignore log_statements: @@ -1739,6 +1863,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_syslog_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -2747,6 +2934,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__custom__debug: exclude: [] fingerprint_size: 5kb @@ -2879,6 +3076,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2916,6 +3119,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra__custom_logging_cassandra__custom__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/otel.yaml index 8389cf9fa2..781e7aae3d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -873,6 +880,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1816,6 +1940,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_syslog_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -3167,6 +3354,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__custom__debug: exclude: [] fingerprint_size: 5kb @@ -3286,6 +3483,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -3379,6 +3582,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra__custom_logging_cassandra__custom__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/otel.yaml index 8389cf9fa2..781e7aae3d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_cassandra_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -873,6 +880,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1816,6 +1940,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "cassandra_syslog_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -3167,6 +3354,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_cassandra__custom__debug: exclude: [] fingerprint_size: 5kb @@ -3286,6 +3483,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -3379,6 +3582,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_cassandra__custom_logging_cassandra__custom__debug: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/otel.yaml index b8b2d33b47..15d759f7ee 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -669,6 +676,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_couchbase__general_1: error_mode: ignore log_statements: @@ -1059,6 +1183,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchbase_http_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1122,6 +1309,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchbase__general: exclude: [] fingerprint_size: 5kb @@ -1212,6 +1409,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchbase_logging_couchbase__general: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/otel.yaml index c2bbee61f1..b121c35fba 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -640,6 +647,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_couchbase__general_1: error_mode: ignore log_statements: @@ -1030,6 +1154,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchbase_http_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1093,6 +1280,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchbase__general: exclude: [] fingerprint_size: 5kb @@ -1181,6 +1378,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchbase_logging_couchbase__general: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/otel.yaml index f0dfe39489..ed4112e16b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1107,6 +1231,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchbase_http_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1513,6 +1700,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchbase__general: exclude: [] fingerprint_size: 5kb @@ -1588,6 +1785,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1669,6 +1872,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchbase_logging_couchbase__general: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/otel.yaml index f0dfe39489..ed4112e16b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1107,6 +1231,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchbase_http_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1513,6 +1700,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchbase__general: exclude: [] fingerprint_size: 5kb @@ -1588,6 +1785,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1669,6 +1872,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchbase_logging_couchbase__general: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/otel.yaml index 99037c6850..2d2ecda397 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_couchdb_1: error_mode: ignore log_statements: @@ -865,6 +989,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchdb") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -928,6 +1115,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchdb: exclude: [] fingerprint_size: 5kb @@ -983,6 +1180,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1008,6 +1211,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchdb_logging_couchdb: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/otel.yaml index 421bc97234..1b864538b5 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_couchdb_1: error_mode: ignore log_statements: @@ -836,6 +960,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchdb") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -899,6 +1086,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchdb: exclude: [] fingerprint_size: 5kb @@ -952,6 +1149,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -977,6 +1180,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchdb_logging_couchdb: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/otel.yaml index 07c1624780..8898997150 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -913,6 +1037,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchdb") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1319,6 +1506,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchdb: exclude: [] fingerprint_size: 5kb @@ -1359,6 +1556,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1440,6 +1643,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchdb_logging_couchdb: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/otel.yaml index 07c1624780..8898997150 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchdb/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -913,6 +1037,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "couchdb") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1319,6 +1506,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_couchdb: exclude: [] fingerprint_size: 5kb @@ -1359,6 +1556,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1440,6 +1643,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_couchdb_logging_couchdb: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf index 412f79898b..cc0eb92e78 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.elasticsearch.elasticsearch_json storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch.elasticsearch_json Name modify @@ -322,30 +300,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf index 30619e8770..316d1bf8eb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf @@ -34,12 +34,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch.elasticsearch_json diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/otel.yaml index 2426f67a97..3855733073 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_elasticsearch__gc_0: error_mode: ignore log_statements: @@ -742,6 +866,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -805,6 +992,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc: exclude: [] fingerprint_size: 5kb @@ -860,6 +1057,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -885,6 +1088,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf index 412f79898b..cc0eb92e78 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.elasticsearch.elasticsearch_json storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch.elasticsearch_json Name modify @@ -322,30 +300,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf index 30619e8770..316d1bf8eb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf @@ -34,12 +34,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch.elasticsearch_json diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/otel.yaml index d7422ebfbd..1a3d39971c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_elasticsearch__gc_0: error_mode: ignore log_statements: @@ -713,6 +837,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -776,6 +963,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc: exclude: [] fingerprint_size: 5kb @@ -829,6 +1026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -854,6 +1057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf index c7d09ee8f2..30bf45c225 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.elasticsearch.elasticsearch_json storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch.elasticsearch_json Name modify @@ -322,30 +300,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf index 30619e8770..316d1bf8eb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf @@ -34,12 +34,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch.elasticsearch_json diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/otel.yaml index 536820a0ae..972ce87cd6 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -790,6 +914,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1196,6 +1383,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc: exclude: [] fingerprint_size: 5kb @@ -1236,6 +1433,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1317,6 +1520,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf index c7d09ee8f2..30bf45c225 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.elasticsearch.elasticsearch_json storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch.elasticsearch_json Name modify @@ -322,30 +300,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf index 30619e8770..316d1bf8eb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf @@ -34,12 +34,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch.elasticsearch_json diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/otel.yaml index 536820a0ae..972ce87cd6 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -790,6 +914,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1196,6 +1383,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc: exclude: [] fingerprint_size: 5kb @@ -1236,6 +1433,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1317,6 +1520,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf index 18d438f6c9..1df7948119 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf @@ -50,13 +50,6 @@ multiline.parser multiline.elasticsearch_default.elasticsearch_json_default storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -72,21 +65,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name modify @@ -571,30 +549,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf index 73352607b4..9b207105ff 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -62,12 +62,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch_custom.elasticsearch_json_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/otel.yaml index bf34d0913a..b1054a7953 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_elasticsearch__gc__custom_0: error_mode: ignore log_statements: @@ -841,6 +965,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc_default") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -904,6 +1091,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc__custom: exclude: [] fingerprint_size: 5kb @@ -975,6 +1172,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1000,6 +1203,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf index 18d438f6c9..1df7948119 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf @@ -50,13 +50,6 @@ multiline.parser multiline.elasticsearch_default.elasticsearch_json_default storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -72,21 +65,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name modify @@ -571,30 +549,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf index 73352607b4..9b207105ff 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf @@ -62,12 +62,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch_custom.elasticsearch_json_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/otel.yaml index 8fab41f299..7283652a9b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_elasticsearch__gc__custom_0: error_mode: ignore log_statements: @@ -812,6 +936,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc_default") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -875,6 +1062,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc__custom: exclude: [] fingerprint_size: 5kb @@ -944,6 +1141,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -969,6 +1172,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf index 3458af52fb..ed84edec43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf @@ -50,13 +50,6 @@ multiline.parser multiline.elasticsearch_default.elasticsearch_json_default storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -72,21 +65,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name modify @@ -571,30 +549,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf index 73352607b4..9b207105ff 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf @@ -62,12 +62,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch_custom.elasticsearch_json_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/otel.yaml index 1addc9be51..00085ae66c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -889,6 +1013,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc_default") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1295,6 +1482,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc__custom: exclude: [] fingerprint_size: 5kb @@ -1351,6 +1548,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1432,6 +1635,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf index 3458af52fb..ed84edec43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf @@ -50,13 +50,6 @@ multiline.parser multiline.elasticsearch_default.elasticsearch_json_default storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -72,21 +65,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name modify @@ -571,30 +549,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf index 73352607b4..9b207105ff 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf @@ -62,12 +62,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.elasticsearch_custom.elasticsearch_json_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/otel.yaml index 1addc9be51..00085ae66c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -889,6 +1013,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "elasticsearch_gc_default") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1295,6 +1482,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_elasticsearch__gc__custom: exclude: [] fingerprint_size: 5kb @@ -1351,6 +1548,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1432,6 +1635,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/otel.yaml index 5c6cafed82..f179f1a622 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -759,6 +883,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id4") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -822,6 +1009,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -932,6 +1129,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -957,6 +1160,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/otel.yaml index 4aa6737078..b87574f1fa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -730,6 +854,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id4") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -793,6 +980,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -901,6 +1098,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -926,6 +1129,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/otel.yaml index 05ee71d40a..c61971185c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id4") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1213,6 +1400,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1308,6 +1505,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1389,6 +1592,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/otel.yaml index 05ee71d40a..c61971185c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_log_file_path/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -807,6 +931,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id4") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1213,6 +1400,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1308,6 +1505,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1389,6 +1592,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml index 3b7214ed6f..13ac359aa7 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -672,6 +796,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -735,6 +922,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/otel.yaml index c059551468..f268c49251 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -643,6 +767,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/otel.yaml index 5e112ead07..b5283396cb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/otel.yaml index 5e112ead07..b5283396cb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_refresh_interval/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -720,6 +844,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id1") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1126,6 +1313,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1167,6 +1364,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/otel.yaml index a9dbc670ff..021adf5a6e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -788,6 +912,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id5") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -912,6 +1099,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1031,6 +1228,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1056,6 +1259,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/otel.yaml index 458a66ae30..bd20e3a520 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_log__source__id1_0: error_mode: ignore log_statements: @@ -759,6 +883,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id5") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -883,6 +1070,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1000,6 +1197,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1025,6 +1228,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/otel.yaml index 9d077e3c53..c178e07ec9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -836,6 +960,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id5") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1303,6 +1490,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1407,6 +1604,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1488,6 +1691,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/otel.yaml index 9d077e3c53..c178e07ec9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_files_type_multiple_receivers/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -836,6 +960,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "log_source_id5") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1303,6 +1490,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_log__source__id1: exclude: [] fingerprint_size: 5kb @@ -1407,6 +1604,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1488,6 +1691,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/otel.yaml index 2c04a9d35c..9659b32e81 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_flink_1: error_mode: ignore log_statements: @@ -767,6 +891,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "flink") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -830,6 +1017,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_flink: exclude: [] fingerprint_size: 5kb @@ -887,6 +1084,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -912,6 +1115,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/otel.yaml index 12ccec13b4..86b8ff1cc2 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_flink_1: error_mode: ignore log_statements: @@ -738,6 +862,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "flink") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_flink: exclude: [] fingerprint_size: 5kb @@ -856,6 +1053,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -881,6 +1084,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/otel.yaml index c9d2dd3d81..bcfbe5ff03 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "flink") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_flink: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/otel.yaml index c9d2dd3d81..bcfbe5ff03 100644 --- a/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_flink/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "flink") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_flink: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/otel.yaml index 9cea5c94aa..32e7a96784 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward_0: error_mode: ignore log_statements: @@ -689,6 +813,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -752,6 +939,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -794,6 +991,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -819,6 +1022,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/otel.yaml index 0a6f3d9580..043a916413 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward_0: error_mode: ignore log_statements: @@ -660,6 +784,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -723,6 +910,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -763,6 +960,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -788,6 +991,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/otel.yaml index 5c818daba0..a374e090cd 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/otel.yaml index 5c818daba0..a374e090cd 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/otel.yaml index 56055c9f7a..08b7a89d04 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward.collision_0: error_mode: ignore log_statements: @@ -735,6 +859,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -798,6 +985,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -842,6 +1039,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -867,6 +1070,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/otel.yaml index ae7c6908e0..00ade83044 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__forward.collision_0: error_mode: ignore log_statements: @@ -706,6 +830,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -769,6 +956,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -811,6 +1008,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -836,6 +1039,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/otel.yaml index c36174ea89..6dc0b4beab 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -783,6 +907,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1189,6 +1376,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 fluentforward/logging_fluent__forward.collision: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/otel.yaml index c36174ea89..6dc0b4beab 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_conflicting_id/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -783,6 +907,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1189,6 +1376,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__forward: endpoint: 1.2.3.4:5678 fluentforward/logging_fluent__forward.collision: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/otel.yaml index 057ff81c0f..92855fb28f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent.forward_0: error_mode: ignore log_statements: @@ -735,6 +859,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -798,6 +985,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -842,6 +1039,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -867,6 +1070,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/otel.yaml index 7e1a11d3d7..cf68ff0ad5 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent.forward_0: error_mode: ignore log_statements: @@ -706,6 +830,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -769,6 +956,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -811,6 +1008,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -836,6 +1039,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/otel.yaml index 811633d54b..adb7760f61 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -783,6 +907,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1189,6 +1376,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent.forward: endpoint: 1.2.3.4:5679 fluentforward/logging_fluent__forward: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/otel.yaml index 811633d54b..adb7760f61 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_multiple_receivers_with_dot/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -783,6 +907,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1189,6 +1376,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent.forward: endpoint: 1.2.3.4:5679 fluentforward/logging_fluent__forward: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/otel.yaml index b6f306adaa..6ec1d42a79 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__logs_0: error_mode: ignore log_statements: @@ -689,6 +813,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -752,6 +939,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -794,6 +991,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -819,6 +1022,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/otel.yaml index e7b8a2cee5..6b1d7f7279 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_fluent__logs_0: error_mode: ignore log_statements: @@ -660,6 +784,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -723,6 +910,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -763,6 +960,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -788,6 +991,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/otel.yaml index 5f7fadf6ad..91c7d8da33 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__logs: endpoint: ":0" hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/otel.yaml index 5f7fadf6ad..91c7d8da33 100644 --- a/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_forward_omitting_optional_parameters/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: statements: - set(attributes["gcp.log_name"], Concat([attributes["gcp.log_name"],body["fluent.tag"]], ".")) where ((attributes != nil and attributes["gcp.log_name"] != nil) and (body != nil and body["fluent.tag"] != nil)) - delete_key(body, "fluent.tag") where ((body != nil and body["fluent.tag"] != nil) and (body != nil and body["fluent.tag"] != nil)) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage fluentforward/logging_fluent__logs: endpoint: ":0" hostmetrics/hostmetrics: @@ -1170,6 +1367,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1251,6 +1454,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/otel.yaml index 9410c2d233..9cdd7c3f98 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hadoop_1: error_mode: ignore log_statements: @@ -769,6 +893,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -832,6 +1019,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -888,6 +1085,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -913,6 +1116,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/otel.yaml index e92b9b57da..4788fe486a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hadoop_1: error_mode: ignore log_statements: @@ -740,6 +864,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -803,6 +990,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -857,6 +1054,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -882,6 +1085,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/otel.yaml index c3c925b618..a2dd0db902 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -817,6 +941,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -1264,6 +1461,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/otel.yaml index c3c925b618..a2dd0db902 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -817,6 +941,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -1264,6 +1461,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/otel.yaml index e40fd4df42..7d58a7e9e4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hadoop_1: error_mode: ignore log_statements: @@ -769,6 +893,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -832,6 +1019,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -889,6 +1086,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -914,6 +1117,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/otel.yaml index f3150809a8..b5eda18e8c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hadoop_1: error_mode: ignore log_statements: @@ -740,6 +864,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -803,6 +990,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -858,6 +1055,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -883,6 +1086,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/otel.yaml index a3be32c6f8..632c61d538 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -817,6 +941,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -1265,6 +1462,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1346,6 +1549,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/otel.yaml index a3be32c6f8..632c61d538 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hadoop_refresh_interval/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -817,6 +941,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hadoop") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hadoop: exclude: [] fingerprint_size: 5kb @@ -1265,6 +1462,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1346,6 +1549,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/otel.yaml index 492d337e09..ae2a8d7c0a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hbase__system_1: error_mode: ignore log_statements: @@ -767,6 +891,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hbase_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -830,6 +1017,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hbase__system: exclude: [] fingerprint_size: 5kb @@ -886,6 +1083,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -911,6 +1114,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/otel.yaml index 55c4cfc002..40ad3b5b55 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_hbase__system_1: error_mode: ignore log_statements: @@ -738,6 +862,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hbase_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hbase__system: exclude: [] fingerprint_size: 5kb @@ -855,6 +1052,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -880,6 +1083,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/otel.yaml index 64ef8d384d..955f07b289 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hbase_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hbase__system: exclude: [] fingerprint_size: 5kb @@ -1262,6 +1459,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1343,6 +1546,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/otel.yaml index 64ef8d384d..955f07b289 100644 --- a/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_hbase/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "hbase_system") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_hbase__system: exclude: [] fingerprint_size: 5kb @@ -1262,6 +1459,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1343,6 +1546,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/otel.yaml index e81ca4419b..b531603b02 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_jetty__access_0: error_mode: ignore log_statements: @@ -817,6 +941,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "jetty_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -880,6 +1067,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_jetty__access: exclude: [] fingerprint_size: 5kb @@ -935,6 +1132,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -960,6 +1163,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/otel.yaml index dd68ac7792..10da553acc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_jetty__access_0: error_mode: ignore log_statements: @@ -788,6 +912,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "jetty_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -851,6 +1038,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_jetty__access: exclude: [] fingerprint_size: 5kb @@ -904,6 +1101,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -929,6 +1132,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/otel.yaml index 2422ecf33a..ff6212718f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -865,6 +989,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "jetty_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_jetty__access: exclude: [] fingerprint_size: 5kb @@ -1311,6 +1508,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1392,6 +1595,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/otel.yaml index 2422ecf33a..ff6212718f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_jetty/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -865,6 +989,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "jetty_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_jetty__access: exclude: [] fingerprint_size: 5kb @@ -1311,6 +1508,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1392,6 +1595,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/otel.yaml index 35d305ff70..1ae36645eb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka_1: error_mode: ignore log_statements: @@ -767,6 +891,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -830,6 +1017,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -887,6 +1084,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -912,6 +1115,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/otel.yaml index b3cf901630..74f931c887 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka_1: error_mode: ignore log_statements: @@ -738,6 +862,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -856,6 +1053,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -881,6 +1084,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/otel.yaml index 030f8f4ab0..5ade14113a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/otel.yaml index 030f8f4ab0..5ade14113a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -815,6 +939,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka: exclude: [] fingerprint_size: 5kb @@ -1263,6 +1460,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1344,6 +1547,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/otel.yaml index b9472fca39..6cdc8bb313 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -669,6 +676,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka__custom_1: error_mode: ignore log_statements: @@ -822,6 +946,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka_syslog") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -967,6 +1154,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka__custom: exclude: [] fingerprint_size: 5kb @@ -1023,6 +1220,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1052,6 +1255,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/otel.yaml index 18d180e971..9147e2f797 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -640,6 +647,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_kafka__custom_1: error_mode: ignore log_statements: @@ -793,6 +917,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka_syslog") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -938,6 +1125,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka__custom: exclude: [] fingerprint_size: 5kb @@ -992,6 +1189,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1021,6 +1224,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/otel.yaml index 2a6768d801..b1b258c6e8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -870,6 +994,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka_syslog") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1358,6 +1545,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka__custom: exclude: [] fingerprint_size: 5kb @@ -1399,6 +1596,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1484,6 +1687,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/otel.yaml index 2a6768d801..b1b258c6e8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_kafka_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -870,6 +994,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "kafka_syslog") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1358,6 +1545,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_kafka__custom: exclude: [] fingerprint_size: 5kb @@ -1399,6 +1596,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1484,6 +1687,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/otel.yaml index 12a24e69f6..d3ac10af79 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mongodb_0: error_mode: ignore log_statements: @@ -985,6 +1109,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1048,6 +1235,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1103,6 +1300,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1128,6 +1331,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/otel.yaml index b2d291bb13..987b3bd0e5 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mongodb_0: error_mode: ignore log_statements: @@ -956,6 +1080,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1019,6 +1206,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1072,6 +1269,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1097,6 +1300,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/otel.yaml index 02b2cfba75..63e4f9e3fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1033,6 +1157,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1439,6 +1626,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1479,6 +1676,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1560,6 +1763,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/otel.yaml index 02b2cfba75..63e4f9e3fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1033,6 +1157,69 @@ processors: - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) - set(body["attributes"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1439,6 +1626,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mongodb: exclude: [] fingerprint_size: 5kb @@ -1479,6 +1676,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1560,6 +1763,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf index 9b018fcb7f..133ba32e66 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf @@ -64,13 +64,6 @@ multiline.parser multiline.mysql.mysql_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -86,21 +79,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql.mysql_error Name lua @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf index cbd5fdaa1f..694a46f385 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf @@ -61,12 +61,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql.mysql_general diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_main.conf index 9b018fcb7f..133ba32e66 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_main.conf @@ -64,13 +64,6 @@ multiline.parser multiline.mysql.mysql_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -86,21 +79,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql.mysql_error Name lua @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_parser.conf index cbd5fdaa1f..694a46f385 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/fluent_bit_parser.conf @@ -61,12 +61,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql.mysql_general diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_main.conf index d19c1ca295..537f0c06e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_main.conf @@ -64,13 +64,6 @@ multiline.parser multiline.mysql.mysql_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -86,21 +79,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql.mysql_error Name lua @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf index cbd5fdaa1f..694a46f385 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf @@ -61,12 +61,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql.mysql_general diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_main.conf index d19c1ca295..537f0c06e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_main.conf @@ -64,13 +64,6 @@ multiline.parser multiline.mysql.mysql_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -86,21 +79,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql.mysql_error Name lua @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_parser.conf index cbd5fdaa1f..694a46f385 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/fluent_bit_parser.conf @@ -61,12 +61,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql.mysql_general diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_main.conf index 2fedcf894f..83a55045a5 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_main.conf @@ -111,13 +111,6 @@ Tag mysql_syslog_error.mysql_syslog_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -133,21 +126,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql_custom.mysql_custom_slow Name modify @@ -621,30 +599,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_parser.conf index a3c0a632ad..a367f5d511 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -257,12 +257,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql_custom.mysql_custom_slow diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/otel.yaml index b15b1ab0ff..37c97f3443 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mysql__custom__error_0: error_mode: ignore log_statements: @@ -1054,6 +1178,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "mysql_custom_general") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1117,6 +1304,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mysql__custom__error: exclude: [] fingerprint_size: 5kb @@ -1189,6 +1386,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1214,6 +1417,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_main.conf index 2fedcf894f..83a55045a5 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_main.conf @@ -111,13 +111,6 @@ Tag mysql_syslog_error.mysql_syslog_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -133,21 +126,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql_custom.mysql_custom_slow Name modify @@ -621,30 +599,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_parser.conf index a3c0a632ad..a367f5d511 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/fluent_bit_parser.conf @@ -257,12 +257,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql_custom.mysql_custom_slow diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/otel.yaml index 3048c915a7..9c153ae3ac 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_mysql__custom__error_0: error_mode: ignore log_statements: @@ -1025,6 +1149,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "mysql_custom_general") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1088,6 +1275,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mysql__custom__error: exclude: [] fingerprint_size: 5kb @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1183,6 +1386,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_main.conf index 7564bb63fc..72cd117c58 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_main.conf @@ -111,13 +111,6 @@ Tag mysql_syslog_error.mysql_syslog_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -133,21 +126,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql_custom.mysql_custom_slow Name modify @@ -621,30 +599,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_parser.conf index a3c0a632ad..a367f5d511 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/fluent_bit_parser.conf @@ -257,12 +257,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql_custom.mysql_custom_slow diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/otel.yaml index db80e6426c..ef9668376b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1102,6 +1226,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "mysql_custom_general") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1508,6 +1695,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mysql__custom__error: exclude: [] fingerprint_size: 5kb @@ -1565,6 +1762,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1646,6 +1849,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_main.conf index 7564bb63fc..72cd117c58 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_main.conf @@ -111,13 +111,6 @@ Tag mysql_syslog_error.mysql_syslog_slow storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -133,21 +126,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match mysql_custom.mysql_custom_slow Name modify @@ -621,30 +599,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_parser.conf index a3c0a632ad..a367f5d511 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/fluent_bit_parser.conf @@ -257,12 +257,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.mysql_custom.mysql_custom_slow diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/otel.yaml index db80e6426c..ef9668376b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mysql_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1102,6 +1226,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "mysql_custom_general") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1508,6 +1695,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_mysql__custom__error: exclude: [] fingerprint_size: 5kb @@ -1565,6 +1762,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1646,6 +1849,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/otel.yaml index 91ba188e67..8ac0f0eb00 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__access_0: error_mode: ignore log_statements: @@ -933,6 +1057,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -996,6 +1183,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1066,6 +1263,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1091,6 +1294,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/otel.yaml index 196b775c97..b98c4af05f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__access_0: error_mode: ignore log_statements: @@ -904,6 +1028,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -967,6 +1154,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1035,6 +1232,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1060,6 +1263,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/otel.yaml index 08bde18bc5..35dac314b9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -981,6 +1105,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1387,6 +1574,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1442,6 +1639,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1523,6 +1726,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/otel.yaml index 08bde18bc5..35dac314b9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -981,6 +1105,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1387,6 +1574,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__access: exclude: [] fingerprint_size: 5kb @@ -1442,6 +1639,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1523,6 +1726,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/otel.yaml index 3a411f0a27..9c3a44a4cb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__custom__access_0: error_mode: ignore log_statements: @@ -1307,6 +1431,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1602,6 +1789,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__custom__access: exclude: [] fingerprint_size: 5kb @@ -1704,6 +1901,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1737,6 +1940,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/otel.yaml index 8e88db015a..ceba91d9c8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_nginx__custom__access_0: error_mode: ignore log_statements: @@ -1278,6 +1402,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1573,6 +1760,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__custom__access: exclude: [] fingerprint_size: 5kb @@ -1673,6 +1870,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1706,6 +1909,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/otel.yaml index fcee2d51b5..d48f1b6a6b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1355,6 +1479,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1993,6 +2180,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__custom__access: exclude: [] fingerprint_size: 5kb @@ -2080,6 +2277,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2169,6 +2372,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/otel.yaml index fcee2d51b5..d48f1b6a6b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_nginx_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -1355,6 +1479,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "nginx_syslog_error") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1993,6 +2180,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_nginx__custom__access: exclude: [] fingerprint_size: 5kb @@ -2080,6 +2277,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2169,6 +2372,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf index 07518bc26f..ce79172b81 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.oracledb.oracledb_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb.oracledb_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf index ff93d827ef..a579e75f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb.oracledb_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/otel.yaml index 8c4714f2d0..9522aeb47c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_oracledb__alert_1: error_mode: ignore log_statements: @@ -758,6 +882,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -821,6 +1008,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -876,6 +1073,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -901,6 +1104,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_main.conf index 07518bc26f..ce79172b81 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.oracledb.oracledb_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb.oracledb_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_parser.conf index ff93d827ef..a579e75f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb.oracledb_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/otel.yaml index 9fd9c930e2..6dec6cd696 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_oracledb__alert_1: error_mode: ignore log_statements: @@ -729,6 +853,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -792,6 +979,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -845,6 +1042,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -870,6 +1073,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf index da2e17ca35..eed1f7a4db 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.oracledb.oracledb_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb.oracledb_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf index ff93d827ef..a579e75f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb.oracledb_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/otel.yaml index e083e7cdd9..bc4a577199 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -806,6 +930,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1212,6 +1399,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1252,6 +1449,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1333,6 +1536,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_main.conf index da2e17ca35..eed1f7a4db 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.oracledb.oracledb_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb.oracledb_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_parser.conf index ff93d827ef..a579e75f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb.oracledb_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/otel.yaml index e083e7cdd9..bc4a577199 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -806,6 +930,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1212,6 +1399,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1252,6 +1449,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1333,6 +1536,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_main.conf index 7263247891..815d96dc11 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_main.conf @@ -59,13 +59,6 @@ Tag oracledb_syslog_audit.oracledb_syslog_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -81,21 +74,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb_custom.oracledb_audit_custom Name modify @@ -237,30 +215,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_parser.conf index 191af997f8..407fc5a9bb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb_custom.oracledb_audit_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/otel.yaml index d5da33234e..f5921e5b45 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_oracledb__alert_1: error_mode: ignore log_statements: @@ -928,6 +1052,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_syslog_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1064,6 +1251,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1134,6 +1331,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1163,6 +1366,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_main.conf index 7263247891..815d96dc11 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_main.conf @@ -59,13 +59,6 @@ Tag oracledb_syslog_audit.oracledb_syslog_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -81,21 +74,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb_custom.oracledb_audit_custom Name modify @@ -237,30 +215,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_parser.conf index 191af997f8..407fc5a9bb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb_custom.oracledb_audit_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/otel.yaml index 7afb1e2278..4db76f1032 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -653,6 +660,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_oracledb__alert_1: error_mode: ignore log_statements: @@ -899,6 +1023,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_syslog_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1035,6 +1222,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1103,6 +1300,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1132,6 +1335,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_main.conf index 93a159a19e..ed91afc6d9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_main.conf @@ -59,13 +59,6 @@ Tag oracledb_syslog_audit.oracledb_syslog_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -81,21 +74,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb_custom.oracledb_audit_custom Name modify @@ -237,30 +215,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_parser.conf index 191af997f8..407fc5a9bb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb_custom.oracledb_audit_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/otel.yaml index ce0857bdcb..b448f7a391 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -976,6 +1100,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_syslog_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1455,6 +1642,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1510,6 +1707,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1595,6 +1798,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_main.conf index 93a159a19e..ed91afc6d9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_main.conf @@ -59,13 +59,6 @@ Tag oracledb_syslog_audit.oracledb_syslog_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -81,21 +74,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match oracledb_custom.oracledb_audit_custom Name modify @@ -237,30 +215,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_parser.conf index 191af997f8..407fc5a9bb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.oracledb_custom.oracledb_audit_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/otel.yaml index ce0857bdcb..b448f7a391 100644 --- a/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_oracledb_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -976,6 +1100,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "oracledb_syslog_alert") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1455,6 +1642,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_oracledb__alert: exclude: [] fingerprint_size: 5kb @@ -1510,6 +1707,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1595,6 +1798,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/otel.yaml index 02dd855225..92daa4019a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__general_1: error_mode: ignore log_statements: @@ -901,6 +1088,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__general: exclude: [] fingerprint_size: 5kb @@ -958,6 +1155,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -983,6 +1186,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/otel.yaml index a1840e36c9..7691361e2c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__general_1: error_mode: ignore log_statements: @@ -872,6 +1059,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__general: exclude: [] fingerprint_size: 5kb @@ -927,6 +1124,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -952,6 +1155,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/otel.yaml index e273115ffc..3f5aacb349 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__general_1: error_mode: ignore log_statements: @@ -1292,6 +1479,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__general: exclude: [] fingerprint_size: 5kb @@ -1334,6 +1531,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1415,6 +1618,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/otel.yaml index e273115ffc..3f5aacb349 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__general_1: error_mode: ignore log_statements: @@ -1292,6 +1479,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__general: exclude: [] fingerprint_size: 5kb @@ -1334,6 +1531,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1415,6 +1618,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/otel.yaml index c2f22c3f4e..6e51fe1cc0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__custom__general_1: error_mode: ignore log_statements: @@ -1304,6 +1491,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__custom__general: exclude: [] fingerprint_size: 5kb @@ -1376,6 +1573,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1405,6 +1608,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/otel.yaml index 5b74f18f73..cbd2fcdb5e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -653,6 +660,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__custom__general_1: error_mode: ignore log_statements: @@ -1275,6 +1462,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__custom__general: exclude: [] fingerprint_size: 5kb @@ -1345,6 +1542,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1374,6 +1577,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/otel.yaml index 874e528ab7..6e4a1ebf3c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -730,6 +854,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__custom__general_1: error_mode: ignore log_statements: @@ -1695,6 +1882,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__custom__general: exclude: [] fingerprint_size: 5kb @@ -1752,6 +1949,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1837,6 +2040,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/otel.yaml index 874e528ab7..6e4a1ebf3c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_postgresql_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -717,6 +724,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -730,6 +854,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_postgresql__custom__general_1: error_mode: ignore log_statements: @@ -1695,6 +1882,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_postgresql__custom__general: exclude: [] fingerprint_size: 5kb @@ -1752,6 +1949,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1837,6 +2040,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/otel.yaml index 3db35306ef..09e85ca31f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_rabbitmq_1: error_mode: ignore log_statements: @@ -890,6 +1077,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_rabbitmq: exclude: [] fingerprint_size: 5kb @@ -945,6 +1142,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -970,6 +1173,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/otel.yaml index 6607f46fcb..5f21beb894 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_rabbitmq_1: error_mode: ignore log_statements: @@ -861,6 +1048,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_rabbitmq: exclude: [] fingerprint_size: 5kb @@ -914,6 +1111,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -939,6 +1142,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/otel.yaml index da8a4858e4..60e2407115 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_rabbitmq_1: error_mode: ignore log_statements: @@ -1281,6 +1468,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_rabbitmq: exclude: [] fingerprint_size: 5kb @@ -1321,6 +1518,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1402,6 +1605,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/otel.yaml index da8a4858e4..60e2407115 100644 --- a/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_rabbitmq/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_rabbitmq_1: error_mode: ignore log_statements: @@ -1281,6 +1468,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_rabbitmq: exclude: [] fingerprint_size: 5kb @@ -1321,6 +1518,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1402,6 +1605,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/otel.yaml index 44bae34518..4724484fcc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -826,6 +1013,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -885,6 +1082,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -910,6 +1113,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/otel.yaml index c06b855926..4a97ceb816 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -797,6 +984,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -854,6 +1051,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -879,6 +1082,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/otel.yaml index afefecac80..d2c8e97e96 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -1217,6 +1404,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -1261,6 +1458,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1342,6 +1545,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/otel.yaml index afefecac80..d2c8e97e96 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis_0: error_mode: ignore log_statements: @@ -1217,6 +1404,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis: exclude: [] fingerprint_size: 5kb @@ -1261,6 +1458,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1342,6 +1545,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/otel.yaml index 46ff5885bd..71b2fd49e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis__custom_0: error_mode: ignore log_statements: @@ -1079,6 +1266,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis__custom: exclude: [] fingerprint_size: 5kb @@ -1154,6 +1351,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1183,6 +1386,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/otel.yaml index 843f2f8f05..a09cd1d17a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis__custom_0: error_mode: ignore log_statements: @@ -1050,6 +1237,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis__custom: exclude: [] fingerprint_size: 5kb @@ -1123,6 +1320,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1152,6 +1355,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/otel.yaml index 3fefb36b94..daed53aa83 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis__custom_0: error_mode: ignore log_statements: @@ -1470,6 +1657,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis__custom: exclude: [] fingerprint_size: 5kb @@ -1530,6 +1727,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1615,6 +1818,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/otel.yaml index 3fefb36b94..daed53aa83 100644 --- a/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_redis_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_redis__custom_0: error_mode: ignore log_statements: @@ -1470,6 +1657,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_redis__custom: exclude: [] fingerprint_size: 5kb @@ -1530,6 +1727,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1615,6 +1818,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf index 372e1aa6a1..ca405fa226 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.saphana.saphana storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana.saphana Name modify @@ -133,30 +111,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf index 0036f583dc..8edc884a43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana.saphana diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_main.conf index 372e1aa6a1..ca405fa226 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.saphana.saphana storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana.saphana Name modify @@ -133,30 +111,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_parser.conf index 0036f583dc..8edc884a43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana.saphana diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_main.conf index 896a00ae9e..185168dd80 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.saphana.saphana storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana.saphana Name modify @@ -133,30 +111,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf index 0036f583dc..8edc884a43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana.saphana diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_main.conf index 896a00ae9e..185168dd80 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_main.conf @@ -34,13 +34,6 @@ multiline.parser multiline.saphana.saphana storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,21 +49,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana.saphana Name modify @@ -133,30 +111,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_parser.conf index 0036f583dc..8edc884a43 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/fluent_bit_parser.conf @@ -14,12 +14,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana.saphana diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_main.conf index 9e9a393d71..e499ba88f4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_main.conf @@ -62,13 +62,6 @@ Tag saphana_syslog.saphana_syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -84,21 +77,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana_custom.saphana_custom Name modify @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_parser.conf index f8e63030e7..3fe12909de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana_custom.saphana_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_main.conf index 9e9a393d71..e499ba88f4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_main.conf @@ -62,13 +62,6 @@ Tag saphana_syslog.saphana_syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -84,21 +77,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana_custom.saphana_custom Name modify @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_parser.conf index f8e63030e7..3fe12909de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana_custom.saphana_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_main.conf index f229f256de..2b1f4c804c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_main.conf @@ -62,13 +62,6 @@ Tag saphana_syslog.saphana_syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -84,21 +77,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana_custom.saphana_custom Name modify @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_parser.conf index f8e63030e7..3fe12909de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana_custom.saphana_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_main.conf index f229f256de..2b1f4c804c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_main.conf @@ -62,13 +62,6 @@ Tag saphana_syslog.saphana_syslog storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -84,21 +77,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match saphana_custom.saphana_custom Name modify @@ -240,30 +218,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_parser.conf index f8e63030e7..3fe12909de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/fluent_bit_parser.conf @@ -35,12 +35,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.saphana_custom.saphana_custom diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_saphana_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/otel.yaml index e4f1a945f7..947006f6fd 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_solr__system_1: error_mode: ignore log_statements: @@ -830,6 +1017,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_solr__system: exclude: [] fingerprint_size: 5kb @@ -885,6 +1082,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -910,6 +1113,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/otel.yaml index 39ff8d1caa..2ea61b2aef 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_solr__system_1: error_mode: ignore log_statements: @@ -801,6 +988,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_solr__system: exclude: [] fingerprint_size: 5kb @@ -854,6 +1051,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -879,6 +1082,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/otel.yaml index 1e46ce2448..79d88e9539 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_solr__system_1: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_solr__system: exclude: [] fingerprint_size: 5kb @@ -1261,6 +1458,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1342,6 +1545,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/otel.yaml index 1e46ce2448..79d88e9539 100644 --- a/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_solr/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_solr__system_1: error_mode: ignore log_statements: @@ -1221,6 +1408,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_solr__system: exclude: [] fingerprint_size: 5kb @@ -1261,6 +1458,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1342,6 +1545,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml index 40ef488271..45c9517ed9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -883,6 +1070,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -907,6 +1104,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -940,6 +1143,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml index f27d9c3a69..ce1ffb7cde 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -854,6 +1041,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -876,6 +1073,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -909,6 +1112,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml index 27b5d5c5db..3a5d630e14 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -937,6 +1124,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -962,6 +1159,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1027,6 +1230,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml index 27b5d5c5db..3a5d630e14 100644 --- a/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_test__syslog__source__id__tcp_0: error_mode: ignore log_statements: @@ -937,6 +1124,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -962,6 +1159,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1027,6 +1230,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_pipeline1_logging_test__syslog__source__id__tcp: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/otel.yaml index 6f6b1693c6..353eb1f650 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -771,6 +958,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -815,6 +1012,12 @@ receivers: storage: file_storage nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -840,6 +1043,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/otel.yaml index 3f7f987bbe..7674e69dfd 100644 --- a/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_systemd/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: priority: debug start_at: beginning storage: file_storage + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_main.conf index 61126e2978..6fa2f358fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_main.conf index 61126e2978..6fa2f358fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_main.conf index 3cdf068947..10c572af10 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_main.conf index 3cdf068947..10c572af10 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_main.conf index 61126e2978..6fa2f358fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_main.conf index 61126e2978..6fa2f358fe 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_main.conf index 3cdf068947..10c572af10 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_main.conf index 3cdf068947..10c572af10 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_duplicated_port_but_not_used/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf index f6c0cbaf93..f3a15fe07c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_main.conf index f6c0cbaf93..f3a15fe07c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf index 929e1b9663..855f5ee8fb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_main.conf index 929e1b9663..855f5ee8fb 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_main.conf @@ -27,13 +27,6 @@ Tag tcp_pipeline.tcp_logs storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -49,21 +42,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match tcp_pipeline.tcp_logs Name lua @@ -90,30 +68,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tcp_omitting_optional_parameters/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/otel.yaml index c0b8a139b5..2ef9458586 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1005,6 +1192,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1078,6 +1275,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1103,6 +1306,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/otel.yaml index cc48485f5f..54ec43a558 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -976,6 +1163,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1047,6 +1244,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1072,6 +1275,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/otel.yaml index 0a725168c0..47f25bb629 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_tomcat__access_0: error_mode: ignore log_statements: @@ -1396,6 +1583,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_tomcat__access: exclude: [] fingerprint_size: 5kb @@ -1454,6 +1651,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1535,6 +1738,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/otel.yaml index 0a725168c0..47f25bb629 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_tomcat__access_0: error_mode: ignore log_statements: @@ -1396,6 +1583,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_tomcat__access: exclude: [] fingerprint_size: 5kb @@ -1454,6 +1651,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1535,6 +1738,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/otel.yaml index ab1dcab960..92990fe709 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -695,6 +702,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1870,6 +2057,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1973,6 +2170,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2006,6 +2209,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/otel.yaml index af9fca7453..d25dcdbfb9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -666,6 +673,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1841,6 +2028,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1942,6 +2139,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1975,6 +2178,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/otel.yaml index e3496f66ac..66b170ebb3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -730,6 +737,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -743,6 +867,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_tomcat__custom__access_0: error_mode: ignore log_statements: @@ -2261,6 +2448,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_tomcat__custom__access: exclude: [] fingerprint_size: 5kb @@ -2349,6 +2546,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2438,6 +2641,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/otel.yaml index e3496f66ac..66b170ebb3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_tomcat_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -730,6 +737,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -743,6 +867,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_tomcat__custom__access_0: error_mode: ignore log_statements: @@ -2261,6 +2448,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_tomcat__custom__access: exclude: [] fingerprint_size: 5kb @@ -2349,6 +2546,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2438,6 +2641,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/otel.yaml index ea0bce3927..743e1abd5f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -880,6 +1067,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -935,6 +1132,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -960,6 +1163,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/otel.yaml index 770d8e1961..0acea9e57a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -851,6 +1038,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -904,6 +1101,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -929,6 +1132,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/otel.yaml index 6393398d9d..df2663b150 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_varnish_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_varnish: exclude: [] fingerprint_size: 5kb @@ -1311,6 +1508,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1392,6 +1595,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/otel.yaml index 6393398d9d..df2663b150 100644 --- a/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_varnish/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_varnish_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_varnish: exclude: [] fingerprint_size: 5kb @@ -1311,6 +1508,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1392,6 +1595,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_main.conf index fb050a5305..8c3a4921be 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.vault.vault_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match vault.vault_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf index 5888ee4c21..1a257edd5a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.vault.vault_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/otel.yaml index d14f09ed76..9e73e649cc 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_main.conf index fb050a5305..8c3a4921be 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.vault.vault_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match vault.vault_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_parser.conf index 5888ee4c21..1a257edd5a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.vault.vault_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/otel.yaml index 3376fff4d4..9d16e006d8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_main.conf index 0bfd50ce31..fbf314f7fa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.vault.vault_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match vault.vault_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_parser.conf index 5888ee4c21..1a257edd5a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.vault.vault_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_main.conf index 0bfd50ce31..fbf314f7fa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_main.conf @@ -33,13 +33,6 @@ multiline.parser multiline.vault.vault_audit storage.type filesystem -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -55,21 +48,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match vault.vault_audit Name modify @@ -132,30 +110,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_parser.conf index 5888ee4c21..1a257edd5a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/fluent_bit_parser.conf @@ -12,12 +12,6 @@ Time_Key time Types severity:string -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time - [MULTILINE_PARSER] flush_timeout 1000 name multiline.vault.vault_audit diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/otel.yaml index 42a760cac7..e6556aa34e 100644 --- a/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_vault/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/otel.yaml index b7237f42b7..2142d8d5c3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -829,6 +1016,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -886,6 +1083,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -911,6 +1114,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/otel.yaml index b93f82ffa4..54b2893cf2 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -800,6 +987,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -855,6 +1052,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -880,6 +1083,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/otel.yaml index 49eb8d09f7..44a17ff8a4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_wildfly__system_1: error_mode: ignore log_statements: @@ -1220,6 +1407,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_wildfly__system: exclude: [] fingerprint_size: 5kb @@ -1262,6 +1459,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1343,6 +1546,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/otel.yaml index 49eb8d09f7..44a17ff8a4 100644 --- a/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_wildfly/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_wildfly__system_1: error_mode: ignore log_statements: @@ -1220,6 +1407,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_wildfly__system: exclude: [] fingerprint_size: 5kb @@ -1262,6 +1459,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1343,6 +1546,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/otel.yaml index 916535cb86..8de01e07d1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -899,6 +1086,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -955,6 +1152,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -980,6 +1183,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/otel.yaml index 2a1e97f7b1..bd11f0b4b8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -870,6 +1057,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -924,6 +1121,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -949,6 +1152,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/otel.yaml index 5d1962d9cc..dde1bb477a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1290,6 +1477,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_zookeeper__general: exclude: [] fingerprint_size: 5kb @@ -1331,6 +1528,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1412,6 +1615,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/otel.yaml index 5d1962d9cc..dde1bb477a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1290,6 +1477,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_zookeeper__general: exclude: [] fingerprint_size: 5kb @@ -1331,6 +1528,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1412,6 +1615,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/otel.yaml index b1f3f2be4a..971c211bdd 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -656,6 +663,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -899,6 +1086,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -954,6 +1151,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -979,6 +1182,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/otel.yaml index 899a2c86d4..37f2d32ee3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -870,6 +1057,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -923,6 +1120,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -948,6 +1151,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/otel.yaml index 451c9ce9c9..6672e030ba 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1290,6 +1477,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_zookeeper__general: exclude: [] fingerprint_size: 5kb @@ -1330,6 +1527,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1411,6 +1614,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/otel.yaml index 451c9ce9c9..6672e030ba 100644 --- a/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_zookeeper_custom/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -691,6 +698,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -704,6 +828,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1290,6 +1477,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_zookeeper__general: exclude: [] fingerprint_size: 5kb @@ -1330,6 +1527,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1411,6 +1614,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/otel.yaml index a5d313c234..21fdee4aea 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/otel.yaml index 8e88eba35e..16402f7f39 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/otel.yaml index f1501c5b21..0a8a590f44 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/otel.yaml index f1501c5b21..0a8a590f44 100644 --- a/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-custom_log_level/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/otel.yaml index bd16edc690..2125c28e0d 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux-gpu/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/otel.yaml index bd16edc690..2125c28e0d 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/otel.yaml index 1a48f998c5..da2121bcac 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/otel.yaml index 1a48f998c5..da2121bcac 100644 --- a/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-default_overrides_disable_all/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml index f8534e886f..d91fb7aaa8 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1007,6 +1194,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1047,6 +1244,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1140,6 +1343,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml index a964c77fed..a46ffb3b4a 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -963,6 +1150,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1001,6 +1198,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1094,6 +1297,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml index c68663a826..274fda153a 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1438,6 +1635,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1587,6 +1790,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml index c68663a826..274fda153a 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1438,6 +1635,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1587,6 +1790,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/otel.yaml index e92525e1b7..59129e07c8 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -619,6 +626,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -682,6 +869,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -720,6 +917,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -745,6 +948,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/otel.yaml index e92525e1b7..59129e07c8 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/linux/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -619,6 +626,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -682,6 +869,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -720,6 +917,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -745,6 +948,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/otel.yaml index 050f0008c2..d1a13b90f2 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows-2012/otel.yaml @@ -77,6 +77,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -693,6 +700,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -706,6 +830,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1112,6 +1299,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1137,6 +1334,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1218,6 +1421,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/otel.yaml index 050f0008c2..d1a13b90f2 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_all_nvml_metrics_individually/golden/windows/otel.yaml @@ -77,6 +77,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -693,6 +700,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -706,6 +830,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1112,6 +1299,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1137,6 +1334,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1218,6 +1421,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/otel.yaml index 2ae40dd1b0..b4fa3844a8 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux-gpu/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -748,6 +945,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -773,6 +976,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/otel.yaml index 6d0c0347cb..2c008cd00e 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/otel.yaml index 9a69181a5c..0b492bd6b4 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/otel.yaml index 9a69181a5c..0b492bd6b4 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_individual/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/otel.yaml index 62db6f16d0..53bb255093 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -710,6 +897,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -750,6 +947,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -775,6 +978,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/otel.yaml index 53c2bf4da1..2303a29e23 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/linux/otel.yaml @@ -54,6 +54,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -679,6 +866,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -717,6 +914,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -742,6 +945,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/otel.yaml index e8eaa5154a..30cb7c423b 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows-2012/otel.yaml @@ -68,6 +68,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -697,6 +821,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1103,6 +1290,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1128,6 +1325,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1209,6 +1412,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/otel.yaml index e8eaa5154a..30cb7c423b 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden/windows/otel.yaml @@ -68,6 +68,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -697,6 +821,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1103,6 +1290,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1128,6 +1325,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1209,6 +1412,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/otel.yaml index f187f70b1a..e83f4f2466 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -710,6 +897,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -750,6 +947,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -775,6 +978,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/otel.yaml index 35cb14fef8..7bcad9b665 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/linux/otel.yaml @@ -54,6 +54,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -679,6 +866,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -717,6 +914,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -742,6 +945,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/otel.yaml index 4d7c1bbee5..7817baa358 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows-2012/otel.yaml @@ -68,6 +68,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -697,6 +821,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1103,6 +1290,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1128,6 +1325,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1209,6 +1412,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/otel.yaml index 4d7c1bbee5..7817baa358 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_globs/golden/windows/otel.yaml @@ -68,6 +68,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -697,6 +821,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1103,6 +1290,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1128,6 +1325,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1209,6 +1412,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/otel.yaml index b6c59f5e2a..220d572966 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux-gpu/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -748,6 +945,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -773,6 +976,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/otel.yaml index 86e9662df7..9a70666030 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/otel.yaml index acce05d9ea..6a40ae2490 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/otel.yaml index acce05d9ea..6a40ae2490 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_metrics_type_filter_with_special_chars/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/otel.yaml index a996e68e62..c388e82fe3 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -747,6 +934,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/otel.yaml index 9c60473fe7..c75f668d68 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/linux/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -653,6 +660,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -716,6 +903,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -758,6 +955,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -783,6 +986,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/otel.yaml index d98ffc0653..1bec8fd7a5 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows-2012/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -721,6 +728,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -734,6 +858,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1169,6 +1366,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/otel.yaml index d98ffc0653..1bec8fd7a5 100644 --- a/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_exclude_workload_metrics/golden/windows/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -721,6 +728,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -734,6 +858,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1169,6 +1366,12 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/otel.yaml index 9812bd2a9d..f27d524f39 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux-gpu/otel.yaml @@ -83,6 +83,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -669,6 +676,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -732,6 +919,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -772,6 +969,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -797,6 +1000,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/otel.yaml index 0e5792b810..b67b466bb1 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/linux/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -627,6 +634,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -690,6 +877,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -728,6 +925,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -753,6 +956,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/otel.yaml index 07e7cc7386..9e8bf56d7f 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1034,6 +1221,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1059,6 +1256,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1108,6 +1311,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/otel.yaml index 07e7cc7386..9e8bf56d7f 100644 --- a/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_three_exclude_metrics_processors_not_disable_nvml/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1034,6 +1221,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1059,6 +1256,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1108,6 +1311,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/otel.yaml index e46552bcce..984f5bdb54 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux-gpu/otel.yaml @@ -71,6 +71,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -657,6 +664,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/otel.yaml index 80c8ccc4a2..0d69e1f3a5 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/linux/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -621,6 +628,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -684,6 +871,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -722,6 +919,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -747,6 +950,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/otel.yaml index b9fd5bcada..9e393fb40f 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows-2012/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -628,6 +635,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1028,6 +1215,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1053,6 +1250,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1102,6 +1305,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/otel.yaml index b9fd5bcada..9e393fb40f 100644 --- a/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-processor_two_exclude_metrics_processors/golden/windows/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -628,6 +635,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1028,6 +1215,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1053,6 +1250,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1102,6 +1305,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/otel.yaml index 32661c8a50..d05ab47b51 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux-gpu/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -633,6 +640,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -696,6 +883,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -736,6 +933,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -761,6 +964,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/otel.yaml index 44b8d6029f..8a392f5b4d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/linux/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -609,6 +616,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -672,6 +859,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -710,6 +907,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -735,6 +938,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/otel.yaml index d0453f95a4..487d391114 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows-2012/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1016,6 +1203,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1041,6 +1238,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1090,6 +1293,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/otel.yaml index d0453f95a4..487d391114 100644 --- a/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver-no-collection_interval/golden/windows/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -616,6 +623,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1016,6 +1203,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1041,6 +1238,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1090,6 +1293,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/otel.yaml index 37638b9578..adad8a09fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/otel.yaml index 2a42221564..abd7920d5b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/otel.yaml index fd09a7f934..bedea1b6b7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/otel.yaml index fd09a7f934..bedea1b6b7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/otel.yaml index 37638b9578..adad8a09fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/otel.yaml index 2a42221564..abd7920d5b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/otel.yaml index fd09a7f934..bedea1b6b7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/otel.yaml index fd09a7f934..bedea1b6b7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_activemq_no_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: activemq username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/otel.yaml index dba50e44dd..80195b6978 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -748,6 +935,16 @@ receivers: password: "" timeout: 20s username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -813,6 +1016,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/otel.yaml index 1ab7b1f371..ffe4cf3650 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -719,6 +906,16 @@ receivers: password: "" timeout: 20s username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -757,6 +954,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -782,6 +985,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/otel.yaml index 5fa9001561..b0bb78138b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ receivers: password: "" timeout: 20s username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1245,6 +1448,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/otel.yaml index 5fa9001561..b0bb78138b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_aerospike/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ receivers: password: "" timeout: 20s username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1245,6 +1448,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/otel.yaml index 1897f6f998..d37efabe23 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux-gpu/otel.yaml @@ -71,6 +71,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -749,6 +936,16 @@ receivers: apache/apache: collection_interval: 60s endpoint: http://127.0.0.1:80/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/otel.yaml index 4ce2135e86..7aa80cbec2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/linux/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -654,6 +661,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ receivers: apache/apache: collection_interval: 60s endpoint: http://127.0.0.1:80/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -758,6 +955,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -783,6 +986,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/otel.yaml index e651c6a6af..e052a3ca99 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows-2012/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -718,6 +725,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -731,6 +855,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ receivers: apache/apache: collection_interval: 60s endpoint: http://127.0.0.1:80/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/otel.yaml index e651c6a6af..e052a3ca99 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache/golden/windows/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -718,6 +725,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -731,6 +855,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ receivers: apache/apache: collection_interval: 60s endpoint: http://127.0.0.1:80/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/otel.yaml index ea24a19ade..6293dcfdbd 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux-gpu/otel.yaml @@ -71,6 +71,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -749,6 +936,16 @@ receivers: apache/apache: collection_interval: 10s endpoint: http://localhost/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/otel.yaml index d8d3c92a78..be29a86dc6 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/linux/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -654,6 +661,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ receivers: apache/apache: collection_interval: 10s endpoint: http://localhost/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -758,6 +955,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -783,6 +986,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/otel.yaml index 154c33f310..7389e9d59e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows-2012/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -718,6 +725,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -731,6 +855,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ receivers: apache/apache: collection_interval: 10s endpoint: http://localhost/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/otel.yaml index 154c33f310..7389e9d59e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_apache_custom/golden/windows/otel.yaml @@ -76,6 +76,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -718,6 +725,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -731,6 +855,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1140,6 +1327,16 @@ receivers: apache/apache: collection_interval: 10s endpoint: http://localhost/server-status?auto + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1165,6 +1362,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/otel.yaml index 18ef3001ee..823e72de07 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/otel.yaml index 48ee735c6b..2e5ee2b4a7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: cassandra,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/otel.yaml index 05d8ff58d0..320bc94263 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: cassandra,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/otel.yaml index 05d8ff58d0..320bc94263 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: cassandra,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/otel.yaml index 9b9dfb2824..8681ad99df 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/otel.yaml index 2695ae99a9..d4b7c27490 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: cassandra username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/otel.yaml index b0d64fdf55..b481923169 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: cassandra username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/otel.yaml index b0d64fdf55..b481923169 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_cassandra_custom/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: cassandra username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/otel.yaml index fcad5e3bea..0cb9670df0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux-gpu/otel.yaml @@ -75,6 +75,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -788,6 +795,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -851,6 +1038,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -891,6 +1088,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -932,6 +1135,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/otel.yaml index 4d49088eaa..358b85095d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/linux/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -759,6 +766,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -822,6 +1009,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -860,6 +1057,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -901,6 +1104,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/otel.yaml index 642ac074e7..832dfcc3f6 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows-2012/otel.yaml @@ -80,6 +80,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -823,6 +830,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -836,6 +960,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1267,6 +1464,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1364,6 +1567,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/otel.yaml index 642ac074e7..832dfcc3f6 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchbase/golden/windows/otel.yaml @@ -80,6 +80,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -823,6 +830,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -836,6 +960,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1242,6 +1429,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1267,6 +1464,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1364,6 +1567,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/otel.yaml index 1d8353d734..2a6b810f70 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ receivers: endpoint: http://localhost:5984 password: otelp username: otelu + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -807,6 +1010,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/otel.yaml index c71dee4aaf..28605c49ee 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -713,6 +900,16 @@ receivers: endpoint: http://localhost:5984 password: otelp username: otelu + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -776,6 +979,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/otel.yaml index 23667f235b..83daf67eeb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ receivers: endpoint: http://localhost:5984 password: otelp username: otelu + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/otel.yaml index 23667f235b..83daf67eeb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_couchdb/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ receivers: endpoint: http://localhost:5984 password: otelp username: otelu + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/otel.yaml index 444637b147..09c6c95cd7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux-gpu/otel.yaml @@ -57,6 +57,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -643,6 +650,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -706,6 +893,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 30s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -771,6 +974,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/otel.yaml index 4a1d115488..9e3316a48c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/linux/otel.yaml @@ -52,6 +52,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -614,6 +621,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -677,6 +864,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -715,6 +912,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -740,6 +943,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/otel.yaml index a3aff3ec93..8c38257471 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 30s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/otel.yaml index a3aff3ec93..8c38257471 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_custom_collection_interval/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 30s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/otel.yaml index fb463a2dc3..a796f08215 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux-gpu/otel.yaml @@ -75,6 +75,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -734,6 +741,186 @@ processors: statements: - set(name, "agent.googleapis.com/dcgm") - set(version, "1.0") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -833,6 +1020,16 @@ receivers: enabled: false gpu.dcgm.xid_errors: enabled: false + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -873,6 +1070,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -898,6 +1101,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/otel.yaml index 06a50af35a..db7382b208 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm/golden/linux/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -705,6 +712,186 @@ processors: statements: - set(name, "agent.googleapis.com/dcgm") - set(version, "1.0") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -804,6 +991,16 @@ receivers: enabled: false gpu.dcgm.xid_errors: enabled: false + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -842,6 +1039,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -867,6 +1070,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/otel.yaml index be683b24e6..15f34c0a6e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -712,6 +719,186 @@ processors: statements: - set(name, "agent.googleapis.com/dcgm") - set(version, "2.0") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -778,6 +965,16 @@ receivers: dcgm/dcgm: collection_interval: 60s endpoint: localhost:5555 + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -818,6 +1015,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -843,6 +1046,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/otel.yaml index 95b93bd038..b7428b0627 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_dcgm_v2/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,186 @@ processors: statements: - set(name, "agent.googleapis.com/dcgm") - set(version, "2.0") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -749,6 +936,16 @@ receivers: dcgm/dcgm: collection_interval: 60s endpoint: localhost:5555 + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -787,6 +984,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -812,6 +1015,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/otel.yaml index 904d800935..8f1d586996 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -780,6 +967,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -820,6 +1017,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -845,6 +1048,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/otel.yaml index a7ff39c439..7e03cce912 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -751,6 +938,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/otel.yaml index 05d16d78cc..6fce04acc3 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/otel.yaml index 05d16d78cc..6fce04acc3 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/otel.yaml index df8227860c..091ff7f13f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -780,6 +967,16 @@ receivers: tls: insecure: true username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -820,6 +1017,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -845,6 +1048,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/otel.yaml index e8be3359b7..e7ff1d5992 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -751,6 +938,16 @@ receivers: tls: insecure: true username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/otel.yaml index 8374b27219..cf4fece5fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/otel.yaml index 8374b27219..cf4fece5fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_credentials/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/otel.yaml index de03a0e62e..4781c3314a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -780,6 +967,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -820,6 +1017,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -845,6 +1048,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/otel.yaml index 4572d4723a..8d0e9e27af 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -751,6 +938,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/otel.yaml index d3550bcd48..5ed93e0a0e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/otel.yaml index d3550bcd48..5ed93e0a0e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_custom_endpoint_http/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/otel.yaml index 8ddb4ee5ce..c825a4d6ea 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -780,6 +967,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -820,6 +1017,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -845,6 +1048,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/otel.yaml index 23b7d19231..9734247a93 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -751,6 +938,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -789,6 +986,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/otel.yaml index c45709c296..c7a03dfefa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/otel.yaml index c45709c296..c7a03dfefa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_disable_cluster_metrics/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1171,6 +1358,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,6 +1393,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/otel.yaml index 95bbb1e134..badc072b50 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -802,6 +989,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -842,6 +1039,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -867,6 +1070,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/otel.yaml index db436920c7..aee13730eb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -773,6 +960,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -811,6 +1008,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -836,6 +1039,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/otel.yaml index b3eb6bf27d..964b7d297e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1193,6 +1380,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/otel.yaml index b3eb6bf27d..964b7d297e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_no_jvm/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1193,6 +1380,16 @@ receivers: tls: insecure: true username: "" + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1218,6 +1415,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1299,6 +1502,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/otel.yaml index 500ec342d4..9e1e317e4c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux-gpu/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -782,6 +969,16 @@ receivers: insecure: false insecure_skip_verify: false username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -822,6 +1019,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -847,6 +1050,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/otel.yaml index b3965d1de9..d1dc8c5307 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/linux/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -649,6 +656,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -753,6 +940,16 @@ receivers: insecure: false insecure_skip_verify: false username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/otel.yaml index da7fa6d413..35f646ee68 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows-2012/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1173,6 +1360,16 @@ receivers: insecure: false insecure_skip_verify: false username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1198,6 +1395,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1279,6 +1482,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/otel.yaml index da7fa6d413..35f646ee68 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_elasticsearch_tls_credentials/golden/windows/otel.yaml @@ -74,6 +74,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -713,6 +720,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -726,6 +850,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1173,6 +1360,16 @@ receivers: insecure: false insecure_skip_verify: false username: user + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1198,6 +1395,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1279,6 +1482,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/otel.yaml index 538a069a3a..49cd9e8ab2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux-gpu/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/otel.yaml index 538a069a3a..49cd9e8ab2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/otel.yaml index 0498223a09..afed5411fd 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/otel.yaml index 0498223a09..afed5411fd 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_nvml_from_hostmetrics/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/otel.yaml index aea0749b2b..19af580df0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux-gpu/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -748,6 +945,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -773,6 +976,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/otel.yaml index 2d306ff3f7..2a50bb35d5 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/otel.yaml index 098dde4cad..df72d025c1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/otel.yaml index 098dde4cad..df72d025c1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_exclude_subset_of_nvml_metrics/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/otel.yaml index 06fa3663f8..41767f2f92 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -706,6 +713,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -769,6 +956,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -812,6 +1009,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -837,6 +1040,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/otel.yaml index eed5acbd3f..f26c35b01f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -677,6 +684,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -740,6 +927,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -781,6 +978,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -806,6 +1009,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/otel.yaml index 64f09cbdde..074570f1f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -741,6 +748,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -754,6 +878,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1160,6 +1347,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage flinkmetrics/flink: collection_interval: 60s endpoint: http://localhost:8081 @@ -1188,6 +1385,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1269,6 +1472,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/otel.yaml index 64f09cbdde..074570f1f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_flink/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -741,6 +748,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -754,6 +878,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1160,6 +1347,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage flinkmetrics/flink: collection_interval: 60s endpoint: http://localhost:8081 @@ -1188,6 +1385,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1269,6 +1472,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/otel.yaml index 1cb663127e..9e21ca639f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/otel.yaml index 87f2d5a929..90023d1d83 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: hadoop,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/otel.yaml index 0282eb9173..8e1415a036 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: hadoop,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/otel.yaml index 0282eb9173..8e1415a036 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: hadoop,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/otel.yaml index fbd385deb0..5cef676d60 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/otel.yaml index 53618df1b9..bf520464dc 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: hadoop username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/otel.yaml index 79d1623786..78bf05fa1d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: hadoop username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/otel.yaml index 79d1623786..78bf05fa1d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hadoop_no_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: hadoop username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/otel.yaml index 20d1d6e163..b55d2cd9d9 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -744,6 +931,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/otel.yaml index e154f52e78..31d60ad3d7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -652,6 +659,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -715,6 +902,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: password: otelp target_system: hbase,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/otel.yaml index 28c68fa8b3..554c66e147 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -716,6 +723,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -729,6 +853,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1167,6 +1364,12 @@ receivers: password: otelp target_system: hbase,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/otel.yaml index 28c68fa8b3..554c66e147 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -716,6 +723,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -729,6 +853,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1167,6 +1364,12 @@ receivers: password: otelp target_system: hbase,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/otel.yaml index 872b8841a7..96bf55f729 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -744,6 +931,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -791,6 +988,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/otel.yaml index 17bd1cb014..62d3475972 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -652,6 +659,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -715,6 +902,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: password: otelp target_system: hbase username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/otel.yaml index 3e0a1d638b..ca1a95a24a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -716,6 +723,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -729,6 +853,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1167,6 +1364,12 @@ receivers: password: otelp target_system: hbase username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/otel.yaml index 3e0a1d638b..ca1a95a24a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_hbase_no_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -716,6 +723,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -729,6 +853,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1167,6 +1364,12 @@ receivers: password: otelp target_system: hbase username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/otel.yaml index e848ca1dd2..46a8f6d6cd 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jetty__metrics_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: target_system: jetty,jvm nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -807,6 +1010,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/otel.yaml index dbce029db3..a98b0e09d7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jetty__metrics_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: endpoint: localhost:1099 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jetty,jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -776,6 +979,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/otel.yaml index e55983b05f..64ada08e45 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:1099 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jetty,jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/otel.yaml index e55983b05f..64ada08e45 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:1099 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jetty,jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/otel.yaml index 93946840ec..8640ea3b92 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jetty__metrics_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/otel.yaml index f49a87455d..55bc30bec9 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jetty__metrics_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: jetty username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/otel.yaml index a712022341..57132e2f4b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: jetty username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/otel.yaml index a712022341..57132e2f4b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jetty_no_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: jetty username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/otel.yaml index 468efeba9d..32c8b39d2d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: target_system: jvm nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -807,6 +1010,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/otel.yaml index ccb0454cbc..1f93790a36 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -776,6 +979,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/otel.yaml index 4f448edff8..ea36252e4c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/otel.yaml index 4f448edff8..ea36252e4c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/otel.yaml index 180c67951d..9352bad72c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otel nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/otel.yaml index c7b9fade3a..3a3978f121 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otel_p target_system: jvm username: otel + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/otel.yaml index 0c39347256..929488d53f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otel_p target_system: jvm username: otel + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/otel.yaml index 0c39347256..929488d53f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_auth/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otel_p target_system: jvm username: otel + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/otel.yaml index c9b38ba895..0a704a3269 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: target_system: jvm nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -807,6 +1010,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/otel.yaml index 3607d3a595..fda9c81ddf 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvm_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -776,6 +979,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/otel.yaml index 83d14e88dd..179c01a949 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/otel.yaml index 83d14e88dd..179c01a949 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_jvm_with_endpoint/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/otel.yaml index 47e65517ed..9d17ff7dc1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/kafka_3: metric_statements: - context: scope @@ -689,6 +813,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -752,6 +939,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -799,6 +996,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -824,6 +1027,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/otel.yaml index c42583d2c7..a6c5de8986 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/kafka_3: metric_statements: - context: scope @@ -660,6 +784,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -723,6 +910,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -768,6 +965,12 @@ receivers: password: otelp target_system: kafka,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -793,6 +996,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/otel.yaml index be31075dbe..1162a07b00 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1175,6 +1372,12 @@ receivers: password: otelp target_system: kafka,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/otel.yaml index be31075dbe..1162a07b00 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1175,6 +1372,12 @@ receivers: password: otelp target_system: kafka,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/otel.yaml index 3c2735bcdb..aa01a29b89 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/kafka_3: metric_statements: - context: scope @@ -689,6 +813,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -752,6 +939,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -799,6 +996,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -824,6 +1027,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/otel.yaml index 5d83d9a9fd..c16a38142e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/kafka_3: metric_statements: - context: scope @@ -660,6 +784,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -723,6 +910,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -768,6 +965,12 @@ receivers: password: otelp target_system: kafka username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -793,6 +996,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/otel.yaml index 2ec9bbb843..56e8a87518 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1175,6 +1372,12 @@ receivers: password: otelp target_system: kafka username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/otel.yaml index 2ec9bbb843..56e8a87518 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_kafka_no_jvm/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -737,6 +861,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1143,6 +1330,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1175,6 +1372,12 @@ receivers: password: otelp target_system: kafka username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/otel.yaml index 8df8af3558..d4832e18b3 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -665,6 +672,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -743,6 +930,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -787,6 +984,12 @@ receivers: transport: tcp nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -812,6 +1015,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/otel.yaml index c5ec6ba6fb..4a4c320cdb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -636,6 +643,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -714,6 +901,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -756,6 +953,12 @@ receivers: collection_interval: 60s endpoint: localhost:11211 transport: tcp + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -781,6 +984,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/otel.yaml index 1a7498baa9..2364bdd1fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -700,6 +707,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -713,6 +837,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1134,6 +1321,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1163,6 +1360,12 @@ receivers: collection_interval: 60s endpoint: localhost:11211 transport: tcp + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1244,6 +1447,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/otel.yaml index 1a7498baa9..2364bdd1fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_memcached/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -700,6 +707,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -713,6 +837,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1134,6 +1321,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1163,6 +1360,12 @@ receivers: collection_interval: 60s endpoint: localhost:11211 transport: tcp + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1244,6 +1447,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/otel.yaml index 7647e1abef..889bf7f293 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -786,6 +983,12 @@ receivers: username: "" nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -811,6 +1014,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/otel.yaml index 0135c75b10..2c44c27566 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -755,6 +952,12 @@ receivers: tls: insecure: false username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -780,6 +983,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/otel.yaml index 710eb562bc..63972d28e4 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: tls: insecure: false username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/otel.yaml index 710eb562bc..63972d28e4 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: tls: insecure: false username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/otel.yaml index bc20e4f3b1..145e1afe66 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: "" nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/otel.yaml index 1389003ffe..59264aaf8e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: transport: unix password: "" username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/otel.yaml index fa87b20499..09b1827eab 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: transport: unix password: "" username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/otel.yaml index fa87b20499..09b1827eab 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mongodb_unix_socket/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: transport: unix password: "" username: "" + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml index 624c066aa4..a2fe00a8e6 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -679,6 +686,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -757,6 +944,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -826,6 +1023,12 @@ receivers: username: usr nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -851,6 +1054,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml index fcc97b45ad..e238958b63 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -650,6 +657,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -728,6 +915,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -795,6 +992,12 @@ receivers: password: pwd transport: tcp username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -820,6 +1023,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml index a580f89181..ead8aa1016 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -714,6 +721,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -727,6 +851,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1148,6 +1335,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1202,6 +1399,12 @@ receivers: password: pwd transport: tcp username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml index a580f89181..ead8aa1016 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -714,6 +721,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -727,6 +851,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1148,6 +1335,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1202,6 +1399,12 @@ receivers: password: pwd transport: tcp username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml index ce9125529f..de47f77260 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -679,6 +686,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -757,6 +944,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -826,6 +1023,12 @@ receivers: username: usr nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -851,6 +1054,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml index 88f42ad2c6..b25e9e80fe 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -650,6 +657,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -728,6 +915,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -795,6 +992,12 @@ receivers: password: pwd transport: unix username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -820,6 +1023,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml index b92f0b136d..f1a9f3fe87 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -714,6 +721,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -727,6 +851,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1148,6 +1335,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1202,6 +1399,12 @@ receivers: password: pwd transport: unix username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml index b92f0b136d..f1a9f3fe87 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -714,6 +721,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -727,6 +851,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1148,6 +1335,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1202,6 +1399,12 @@ receivers: password: pwd transport: unix username: usr + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/otel.yaml index 05e54665c4..82814227ac 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -780,6 +977,12 @@ receivers: endpoint: http://127.0.0.1/status nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -805,6 +1008,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/otel.yaml index 937db0e665..24adb02b70 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -749,6 +946,12 @@ receivers: nginx/nginx: collection_interval: 60s endpoint: http://127.0.0.1/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -774,6 +977,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/otel.yaml index 20fcda9306..96ef6bdc13 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: nginx/nginx: collection_interval: 60s endpoint: http://127.0.0.1/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/otel.yaml index 20fcda9306..96ef6bdc13 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: nginx/nginx: collection_interval: 60s endpoint: http://127.0.0.1/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/otel.yaml index da2b3c6ea1..82580fdf3e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -780,6 +977,12 @@ receivers: endpoint: http://localhost/status nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -805,6 +1008,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/otel.yaml index bd5546d9a7..e16b7a235b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -749,6 +946,12 @@ receivers: nginx/nginx: collection_interval: 30s endpoint: http://localhost/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -774,6 +977,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/otel.yaml index 7f0065c9ee..61b076b0cb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: nginx/nginx: collection_interval: 30s endpoint: http://localhost/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/otel.yaml index 7f0065c9ee..61b076b0cb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_nginx_custom/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1156,6 +1353,12 @@ receivers: nginx/nginx: collection_interval: 30s endpoint: http://localhost/status + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/otel.yaml index a66a2cb36c..8df6d6d456 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -762,6 +949,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -802,6 +999,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1422,6 +1625,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/otel.yaml index 84cb87339a..a136431ef5 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -655,6 +662,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -733,6 +920,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -771,6 +968,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1391,6 +1594,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/otel.yaml index 4c1e8b59ff..7029e50cad 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/otel.yaml index 4c1e8b59ff..7029e50cad 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/otel.yaml index a20230ac85..1d67102012 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -762,6 +949,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -802,6 +999,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1422,6 +1625,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/otel.yaml index 2527f5155b..8cdf6411bb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -655,6 +662,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -733,6 +920,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -771,6 +968,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1391,6 +1594,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/otel.yaml index 8130771d35..4b54d5cba0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/otel.yaml index 8130771d35..4b54d5cba0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_all_params/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/otel.yaml index 0764629f62..1483c43c43 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -684,6 +691,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -762,6 +949,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -802,6 +999,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1422,6 +1625,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/otel.yaml index 6723956625..7388d54e3e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -655,6 +662,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -733,6 +920,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -771,6 +968,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1391,6 +1594,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/otel.yaml index 168071e611..91ad3ea596 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/otel.yaml index 168071e611..91ad3ea596 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_oracledb_unix_socket/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -719,6 +726,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -732,6 +856,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1153,6 +1340,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1178,6 +1375,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1854,6 +2057,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/otel.yaml index 0628cfb4c1..2fa963f376 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -748,6 +935,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -824,6 +1027,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/otel.yaml index 73448f2312..9cfc09b074 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -719,6 +906,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -757,6 +954,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -793,6 +996,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/otel.yaml index 6af7fe2adb..6c62f7401a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/otel.yaml index 6af7fe2adb..6c62f7401a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1256,6 +1459,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/otel.yaml index 14307579f6..0903be9dda 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -748,6 +935,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -826,6 +1029,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/otel.yaml index 3dc5e9dd47..82cfdffd99 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -719,6 +906,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -757,6 +954,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -795,6 +998,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/otel.yaml index 25b93e4210..dbd3ecaa2b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1258,6 +1461,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/otel.yaml index 25b93e4210..dbd3ecaa2b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1258,6 +1461,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/otel.yaml index 84df41b78f..7936a6a1fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -748,6 +935,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -827,6 +1030,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/otel.yaml index cc3543e81a..50167e15f4 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -719,6 +906,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -757,6 +954,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -796,6 +999,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/otel.yaml index a28132074c..1106c93cff 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1259,6 +1462,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/otel.yaml index a28132074c..1106c93cff 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_no_sni/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1259,6 +1462,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/otel.yaml index 5f692c8fe7..806e671ee6 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -748,6 +935,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -788,6 +985,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -830,6 +1033,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/otel.yaml index fc79296478..6e8b291aae 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -634,6 +641,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -719,6 +906,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -757,6 +954,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -799,6 +1002,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/otel.yaml index a300890600..8b10ad16d0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1262,6 +1465,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/otel.yaml index a300890600..8b10ad16d0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_postgresql_tls_with_certs/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -698,6 +705,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -711,6 +835,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1139,6 +1326,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1164,6 +1361,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1262,6 +1465,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/otel.yaml index 48189eda32..84cf322475 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -852,6 +1055,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/otel.yaml index 8408988694..2ff5bb4338 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -821,6 +1024,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/otel.yaml index 29f6fc2272..542f1a2348 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/otel.yaml index 29f6fc2272..542f1a2348 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/otel.yaml index 2ba325c90c..6260ecdb35 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -855,6 +1058,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/otel.yaml index 0d528ff7a6..519eb9d706 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -824,6 +1027,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/otel.yaml index ecf8bf78c8..adf330c76c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1287,6 +1490,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/otel.yaml index ecf8bf78c8..adf330c76c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_basic_auth/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1287,6 +1490,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/otel.yaml index 23ba0e97ee..86bf89a1f8 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -916,6 +1119,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/otel.yaml index 70954e36f8..92e800e95c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -885,6 +1088,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/otel.yaml index 16bb82c0d4..efecf76003 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1348,6 +1551,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/otel.yaml index 16bb82c0d4..efecf76003 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_complex/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1348,6 +1551,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/otel.yaml index 63db2f8b96..e2cbffb2c4 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -851,6 +1054,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/otel.yaml index f0b2e4e8bb..80ce178cea 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -820,6 +1023,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/otel.yaml index a2227f0793..fcb3a1dc90 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/otel.yaml index a2227f0793..fcb3a1dc90 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_default_replacement/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1283,6 +1486,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/otel.yaml index 2d70663511..72a2fd3715 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -852,6 +1055,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/otel.yaml index 7718d80e52..ae1eebe868 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -821,6 +1024,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/otel.yaml index fb51260800..cedc3de2b2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/otel.yaml index fb51260800..cedc3de2b2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_multi_replacement_regex/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/otel.yaml index c4beeb26b0..4ebf947441 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -845,6 +1048,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/otel.yaml index defc3ae0ef..d86de3a2ef 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/otel.yaml index d608917a31..ef44eeb6d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/otel.yaml index d608917a31..ef44eeb6d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_node_exporter/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1277,6 +1480,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/otel.yaml index cb1db442f8..6f6e4c7461 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -906,6 +1109,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/otel.yaml index bb1f9e839f..c2cd226d11 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -875,6 +1078,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/otel.yaml index 70c5c4a32a..3447f8a68d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1338,6 +1541,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/otel.yaml index 70c5c4a32a..3447f8a68d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_relabel/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1338,6 +1541,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/otel.yaml index eb5583af17..608e014bdf 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -852,6 +1055,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/otel.yaml index e3f45e00af..f78c4e07b9 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -821,6 +1024,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/otel.yaml index be29a971d8..e935d4241a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/otel.yaml index be29a971d8..e935d4241a 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_replace_using_capture_groups/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1284,6 +1487,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/otel.yaml index 038d641751..3128b99870 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -888,6 +1091,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/otel.yaml index f33f10ef8a..1c6f902f60 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -857,6 +1060,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/otel.yaml index d47120b4f3..4cb6ab2d39 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1320,6 +1523,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/otel.yaml index d47120b4f3..4cb6ab2d39 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_scrape_interval/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1320,6 +1523,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/otel.yaml index eaa28644cf..792ee20680 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux-gpu/otel.yaml @@ -61,6 +61,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -647,6 +654,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -720,6 +907,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -760,6 +957,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -850,6 +1053,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/otel.yaml index 22da6e0d25..2d9ba8dc25 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/linux/otel.yaml @@ -56,6 +56,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -618,6 +625,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -691,6 +878,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -729,6 +926,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -819,6 +1022,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/otel.yaml index b1b075b635..b727a441ef 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows-2012/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1282,6 +1485,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/otel.yaml index b1b075b635..b727a441ef 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_prometheus_tlx_with_certs/golden/windows/otel.yaml @@ -66,6 +66,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -682,6 +689,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -695,6 +819,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1111,6 +1298,16 @@ processors: - delete_key(attributes, "cluster") - delete_key(attributes, "namespace") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1136,6 +1333,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1282,6 +1485,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/otel.yaml index 7c43c5c25b..58bbbf0009 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/otel.yaml index 528a43e49e..c7d1601b9b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -713,6 +900,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -783,6 +986,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/otel.yaml index 448990c9d6..f9310a1fc9 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/otel.yaml index 448990c9d6..f9310a1fc9 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/otel.yaml index da6db68ad5..c447196f0d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -814,6 +1017,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/otel.yaml index 740d7cce83..d3f28963ea 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -713,6 +900,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -783,6 +986,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/otel.yaml index 4c88fd0830..9da23a7616 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/otel.yaml index 4c88fd0830..9da23a7616 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1246,6 +1449,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/otel.yaml index 72946a8683..a4b54d84e7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -815,6 +1018,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/otel.yaml index d2a8b6795f..3560443f86 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -713,6 +900,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -784,6 +987,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/otel.yaml index 6ed5c2c56c..a05015c08d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1247,6 +1450,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/otel.yaml index 6ed5c2c56c..a05015c08d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_no_sni/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1247,6 +1450,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/otel.yaml index bd1602a7d8..74d6e14558 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -742,6 +929,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -818,6 +1021,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/otel.yaml index d44df12993..fe4df4fc33 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -713,6 +900,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -787,6 +990,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/otel.yaml index afa35571d3..baa8dd63e1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/otel.yaml index afa35571d3..baa8dd63e1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_rabbitmq_tls_with_certs/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/otel.yaml index 7f525313eb..0c24f0a78c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -666,6 +673,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -744,6 +931,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/otel.yaml index fc0e5110f9..c805f23b65 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -637,6 +644,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -715,6 +902,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/otel.yaml index 4e546e1895..6b6d833157 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/otel.yaml index 4e546e1895..6b6d833157 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/otel.yaml index c655943102..4292e28b81 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -666,6 +673,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -744,6 +931,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -816,6 +1019,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/otel.yaml index abf91f404d..a892603fcf 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -637,6 +644,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -715,6 +902,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -785,6 +988,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/otel.yaml index 2a5e4d2bea..a5ba633135 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/otel.yaml index 2a5e4d2bea..a5ba633135 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_redis_custom/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -701,6 +708,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -714,6 +838,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1135,6 +1322,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1248,6 +1451,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/otel.yaml index fffa7f849b..c9029c0914 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -665,6 +672,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -746,6 +933,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -786,6 +983,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -818,6 +1021,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/otel.yaml index 8d366a8267..1537ee2bd0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -636,6 +643,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -717,6 +904,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -755,6 +952,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -787,6 +990,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/otel.yaml index 5b02bdb2db..c94d461f19 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -700,6 +707,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -713,6 +837,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1137,6 +1324,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/otel.yaml index 5b02bdb2db..c94d461f19 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_saphana/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -700,6 +707,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -713,6 +837,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1137,6 +1324,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1250,6 +1453,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/otel.yaml index febc63626f..11d9945a91 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/otel.yaml index aeeb85aa46..96be80d304 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: solr username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/otel.yaml index 91d642a15d..fb1ea2206e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: solr username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/otel.yaml index 91d642a15d..fb1ea2206e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_solr/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: solr username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/otel.yaml index 5ca84a8b77..72343d6451 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/otel.yaml index bfa3ddb822..cac171b253 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: tomcat,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/otel.yaml index dd8cb25864..6c05b4cc27 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: tomcat,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/otel.yaml index dd8cb25864..6c05b4cc27 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: tomcat,jvm username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/otel.yaml index 16ea4ed10a..a3466f5b6e 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -784,6 +981,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -809,6 +1012,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/otel.yaml index c89e7b66d7..f955a17850 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -753,6 +950,12 @@ receivers: password: otelp target_system: tomcat username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -778,6 +981,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/otel.yaml index 4050a7d09f..ab8b27d748 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: tomcat username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/otel.yaml index 4050a7d09f..ab8b27d748 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_tomcat_custom/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1160,6 +1357,12 @@ receivers: password: otelp target_system: tomcat username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/otel.yaml index 80c034e94f..873059d2f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -777,6 +974,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -806,6 +1009,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/otel.yaml index aa57307a28..8bdd04e3e7 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -775,6 +978,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/otel.yaml index fab36439af..e41db01b15 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1238,6 +1441,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/otel.yaml index fab36439af..e41db01b15 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_varnish/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1238,6 +1441,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/otel.yaml index 51b028dc9b..65c5d8e84b 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1453,6 +1650,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1489,6 +1692,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/otel.yaml index 450df86b4a..2e4bcc6700 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1384,6 +1571,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1422,6 +1619,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1458,6 +1661,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/otel.yaml index f0b8f6cb3c..ce921edc35 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1921,6 +2124,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/otel.yaml index f0b8f6cb3c..ce921edc35 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1921,6 +2124,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/otel.yaml index edbfe6aff8..c80226dd6f 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1453,6 +1650,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1489,6 +1692,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/otel.yaml index 4c75041586..d7ef3a35be 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1384,6 +1571,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1422,6 +1619,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1458,6 +1661,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/otel.yaml index d6c773a552..a9ecf19ddc 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1921,6 +2124,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/otel.yaml index d6c773a552..a9ecf19ddc 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_tls/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1921,6 +2124,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/otel.yaml index 59b2812077..129c1cf5a2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -703,6 +710,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1453,6 +1650,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1493,6 +1696,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/otel.yaml index 7713f3845b..54959f54c1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -674,6 +681,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1384,6 +1571,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1422,6 +1619,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1462,6 +1665,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/otel.yaml index 2bd229b8f2..06362de05c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1925,6 +2128,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/otel.yaml index 2bd229b8f2..06362de05c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_vault_with_token/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -738,6 +745,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -751,6 +875,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1804,6 +1991,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1829,6 +2026,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1925,6 +2128,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/otel.yaml index 4b26653cbd..18a6aa15fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -786,6 +983,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -811,6 +1014,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/otel.yaml index 3bfc11a24c..97856081d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -755,6 +952,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -780,6 +983,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/otel.yaml index fc4cb01655..b740d8198c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/otel.yaml index fc4cb01655..b740d8198c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/otel.yaml index 4b26653cbd..18a6aa15fa 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -786,6 +983,12 @@ receivers: username: otelu nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -811,6 +1014,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/otel.yaml index 3bfc11a24c..97856081d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -755,6 +952,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -780,6 +983,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/otel.yaml index fc4cb01655..b740d8198c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/otel.yaml index fc4cb01655..b740d8198c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_wildfly_with_host_port/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1162,6 +1359,12 @@ receivers: password: otelp target_system: wildfly username: otelu + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1243,6 +1446,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/otel.yaml index 82f73a9df8..eecdd899cb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -777,6 +974,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -805,6 +1008,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/otel.yaml index 8aa25037d3..b9d94dd26d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -774,6 +977,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/otel.yaml index 0dac8541c5..6e0e6645f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/otel.yaml index 0dac8541c5..6e0e6645f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/otel.yaml index 82f73a9df8..eecdd899cb 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -777,6 +974,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -805,6 +1008,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/otel.yaml index 8aa25037d3..b9d94dd26d 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -746,6 +943,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -774,6 +977,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/otel.yaml index 0dac8541c5..6e0e6645f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/otel.yaml index 0dac8541c5..6e0e6645f0 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_zookeeper_endpoint/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -707,6 +831,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1153,6 +1350,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1237,6 +1440,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/otel.yaml index 60bf31ed2e..886cceb855 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux-gpu/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -983,6 +1170,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1023,6 +1220,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1049,6 +1252,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/otel.yaml b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/otel.yaml index b8eb67c48e..7f99ad70b5 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/linux/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -939,6 +1126,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -977,6 +1174,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1003,6 +1206,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/otel.yaml index c9295063d6..845993c588 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows-2012/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1389,6 +1576,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1414,6 +1611,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1496,6 +1699,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/otel.yaml b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/otel.yaml index c9295063d6..845993c588 100644 --- a/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/otlp_exporter_global_config/golden/windows/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1389,6 +1576,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1414,6 +1611,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1496,6 +1699,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/otel.yaml index 7683879a87..d9f52d68d0 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux-gpu/otel.yaml @@ -64,6 +64,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -860,6 +867,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_1_1: metric_statements: - context: scope @@ -890,6 +1014,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -1007,6 +1194,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1047,6 +1244,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1182,6 +1385,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/otel.yaml b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/otel.yaml index bd85c7552f..2340fc5f7f 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/linux/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -831,6 +838,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -846,6 +970,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -963,6 +1150,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -1001,6 +1198,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1136,6 +1339,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/otel.yaml index 719f8c655e..8dfa584062 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows-2012/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1438,6 +1635,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1629,6 +1832,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/otel.yaml b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/otel.yaml index 719f8c655e..8dfa584062 100644 --- a/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/test-otlp-exporter-only/golden/windows/otel.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1413,6 +1600,16 @@ processors: statements: - set(metric.name, Concat(["prometheus.googleapis.com/", metric.name], "")) where not HasPrefix(metric.name, "prometheus.googleapis.com/") receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1438,6 +1635,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1629,6 +1832,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml index d9fd689a62..1ab31ea05a 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -676,6 +800,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1082,6 +1269,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1107,6 +1304,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1188,6 +1391,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml index d9fd689a62..1ab31ea05a 100644 --- a/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -676,6 +800,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1082,6 +1269,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1107,6 +1304,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1188,6 +1391,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/otel.yaml index 42acdda8ef..d0b2fddeb6 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/otel.yaml index 42acdda8ef..d0b2fddeb6 100644 --- a/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-all-custom_use_built_in_receivers/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1097,6 +1284,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1122,6 +1319,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1203,6 +1406,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/otel.yaml index f368c70033..7442d9379c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -975,6 +1099,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "active_directory_ds") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1381,6 +1568,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1406,6 +1603,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1503,6 +1706,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_active__directory__ds_logging_active__directory__ds: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/otel.yaml index f368c70033..7442d9379c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_active_directory_ds/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -975,6 +1099,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "active_directory_ds") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1381,6 +1568,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1406,6 +1603,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1503,6 +1706,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_active__directory__ds_logging_active__directory__ds: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/otel.yaml index e81d4ef562..b478d1478d 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -882,6 +1006,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "iis_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1288,6 +1475,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_iis__access: exclude: [] fingerprint_size: 5kb @@ -1328,6 +1525,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1409,6 +1612,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/otel.yaml index e81d4ef562..b478d1478d 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_iis/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -683,6 +690,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -882,6 +1006,69 @@ processors: - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) - set(cache["value"], "iis_access") where cache["value"] == nil - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1288,6 +1475,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_iis__access: exclude: [] fingerprint_size: 5kb @@ -1328,6 +1525,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1409,6 +1612,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml index e6ee7b7430..2914ef9d8d 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1619,6 +1806,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1644,6 +1841,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1749,6 +1952,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml index e6ee7b7430..2914ef9d8d 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1619,6 +1806,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1644,6 +1841,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1749,6 +1952,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/otel.yaml index fffee67524..3e6963acfb 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1253,6 +1440,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1278,6 +1475,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1359,6 +1562,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/otel.yaml index fffee67524..3e6963acfb 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_override_builtin/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1253,6 +1440,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1278,6 +1475,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1359,6 +1562,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml index a0b54a3964..99d17b06b6 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1248,6 +1445,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/otel.yaml index a0b54a3964..99d17b06b6 100644 --- a/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-receiver_winlog2_xml/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1248,6 +1445,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/otel.yaml index e8860974b3..9b46c9cccf 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1296,6 +1493,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1385,6 +1588,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/otel.yaml index e8860974b3..9b46c9cccf 100644 --- a/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-logging-use_ansi/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1271,6 +1458,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1296,6 +1493,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1385,6 +1588,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/otel.yaml index c485d03f52..6b08d2c066 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux-gpu/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -748,6 +945,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -773,6 +976,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/otel.yaml index d397260d97..03576d07b3 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/otel.yaml index 08febd6a7d..8a8ddf7975 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/otel.yaml index 08febd6a7d..8a8ddf7975 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_iis/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/otel.yaml index fe5ea4a6d1..1048881822 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux-gpu/otel.yaml @@ -59,6 +59,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -645,6 +652,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -748,6 +945,12 @@ receivers: processes: {} nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -773,6 +976,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/otel.yaml index 9aafb2c188..2ec74de391 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/linux/otel.yaml @@ -53,6 +53,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -615,6 +622,186 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -678,6 +865,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -716,6 +913,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -741,6 +944,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/otel.yaml index 2e3463c9b3..4ab537c9cf 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows-2012/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/otel.yaml index 2e3463c9b3..4ab537c9cf 100644 --- a/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-default_overrides_disable_mssql/golden/windows/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1100,6 +1287,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1125,6 +1322,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml index 74cbb1a32f..fc4cde1fc9 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows-2012/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -676,6 +800,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1082,6 +1269,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1107,6 +1304,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1188,6 +1391,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/otel.yaml index 74cbb1a32f..fc4cde1fc9 100644 --- a/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-pipeline_multiple_pipelines/golden/windows/otel.yaml @@ -47,6 +47,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -663,6 +670,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -676,6 +800,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1082,6 +1269,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1107,6 +1304,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1188,6 +1391,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/otel.yaml index 821d8b4f4d..6251fa7887 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1130,6 +1317,16 @@ processors: receivers: active_directory_ds/active__directory__ds: collection_interval: 60s + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1155,6 +1352,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1236,6 +1439,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/otel.yaml index 821d8b4f4d..6251fa7887 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_active_directory_ds/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -709,6 +716,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1130,6 +1317,16 @@ processors: receivers: active_directory_ds/active__directory__ds: collection_interval: 60s + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1155,6 +1352,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1236,6 +1439,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/otel.yaml index a82dea469b..7adbf9bf04 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -741,6 +865,69 @@ processors: - context: datapoint statements: - keep_keys(resource.attributes, []) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1147,6 +1334,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1174,6 +1371,12 @@ receivers: processes: {} iis/iis__v2: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1255,6 +1458,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/otel.yaml index a82dea469b..7adbf9bf04 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_duplicate/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -704,6 +711,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -741,6 +865,69 @@ processors: - context: datapoint statements: - keep_keys(resource.attributes, []) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1147,6 +1334,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1174,6 +1371,12 @@ receivers: processes: {} iis/iis__v2: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1255,6 +1458,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/otel.yaml index 6cba99653d..97488b69dd 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -672,6 +679,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_0: metric_statements: - context: datapoint @@ -696,6 +820,69 @@ processors: - context: datapoint statements: - keep_keys(resource.attributes, []) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1102,6 +1289,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1129,6 +1326,12 @@ receivers: processes: {} iis/iis: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1192,6 +1395,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/otel.yaml index 6cba99653d..97488b69dd 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_iis_v2_override/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -672,6 +679,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_0: metric_statements: - context: datapoint @@ -696,6 +820,69 @@ processors: - context: datapoint statements: - keep_keys(resource.attributes, []) + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1102,6 +1289,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1129,6 +1326,12 @@ receivers: processes: {} iis/iis: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1192,6 +1395,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/otel.yaml index beb00e8738..107ebc3b83 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux-gpu/otel.yaml @@ -65,6 +65,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -659,6 +666,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvmmetrics_2: metric_statements: - context: scope @@ -674,6 +798,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -737,6 +924,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -782,6 +979,12 @@ receivers: target_system: jvm nvml/hostmetrics_1: collection_interval: 60s + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -807,6 +1010,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_main.conf index 9d767d886b..baf8a4c527 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/otel.yaml index d8d3e15443..b260961492 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/linux/otel.yaml @@ -60,6 +60,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -630,6 +637,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/jvmmetrics_2: metric_statements: - context: scope @@ -645,6 +769,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_syslog_0: error_mode: ignore log_statements: @@ -708,6 +895,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage file_log/logging_syslog: exclude: [] fingerprint_size: 5kb @@ -751,6 +948,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -776,6 +979,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_syslog: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/otel.yaml index d604f6ae55..4ea7e91b42 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/otel.yaml index d604f6ae55..4ea7e91b42 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_jvm_missing_endpoint/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -694,6 +701,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -722,6 +846,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1128,6 +1315,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: endpoint: localhost:9999 jar_path: /path/to/executables/opentelemetry-java-contrib-jmx-metrics.jar target_system: jvm + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1239,6 +1442,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/otel.yaml index a12e464108..3709380702 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -696,6 +703,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -709,6 +833,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/otel.yaml index a12e464108..3709380702 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_duplicate/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -696,6 +703,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -709,6 +833,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1133,6 +1320,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1158,6 +1355,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1241,6 +1444,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/otel.yaml index 355e858440..1f92d1c627 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows-2012/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1112,6 +1299,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1137,6 +1334,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/otel.yaml index 355e858440..1f92d1c627 100644 --- a/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-metrics-receiver_mssql_v2_override/golden/windows/otel.yaml @@ -70,6 +70,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -681,6 +688,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -694,6 +818,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1112,6 +1299,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1137,6 +1334,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1206,6 +1409,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml index e6ee7b7430..2914ef9d8d 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1619,6 +1806,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1644,6 +1841,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1749,6 +1952,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel_otlp_exporter.yaml index 842102a451..8b0261527a 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1911,6 +2098,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1936,6 +2133,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2042,6 +2245,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml index e6ee7b7430..2914ef9d8d 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1619,6 +1806,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1644,6 +1841,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1749,6 +1952,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel_otlp_exporter.yaml index 842102a451..8b0261527a 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1911,6 +2098,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1936,6 +2133,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -2042,6 +2245,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml index a0b54a3964..99d17b06b6 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1248,6 +1445,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel_otlp_exporter.yaml index 52996669c9..9b251cbbb2 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1515,6 +1702,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1540,6 +1737,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1638,6 +1841,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf index a8cb802c02..ce41551674 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_main.conf @@ -17,13 +17,6 @@ Scrape_Interval 60 Scrape_On_Start True -[INPUT] - Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} - Interval_NSec 0 - Interval_Sec 600 - Name dummy - Tag ops-agent-health - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -39,21 +32,6 @@ Tag ops-agent-fluent-bit storage.type memory -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/ops-agent-health - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path ${logs_dir}/health-checks.log - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag ops-agent-health - storage.type memory - [FILTER] Match ops-agent-fluent-bit Name lua @@ -74,30 +52,6 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua -[FILTER] - Match ops-agent-health - Name lua - call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua - -[FILTER] - Key_Name message - Match ops-agent-health - Name parser - Reserve_Data True - Parser ops-agent-health.health-checks-json - -[FILTER] - Match ops-agent-health - Name lua - call parser_merge_record - script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua - -[FILTER] - Match ops-agent-health - Name grep - Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug - [FILTER] Match ops-agent-fluent-bit Name rewrite_tag diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf index 3c603780bd..7c1a7e754c 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/fluent_bit_parser.conf @@ -5,9 +5,3 @@ Time_Format %Y/%m/%d %H:%M:%S Time_Key time Types severity:string - -[PARSER] - Format json - Name ops-agent-health.health-checks-json - Time_Format %Y-%m-%dT%H:%M:%S%z - Time_Key time diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json new file mode 100644 index 0000000000..9a3261ba40 --- /dev/null +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/logging_ping_otlp.json @@ -0,0 +1 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"kvlistValue":{"values":[{"key":"code","value":{"stringValue":"LogPingOpsAgent"}},{"key":"severity","value":{"stringValue":"DEBUG"}}]}}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml index a0b54a3964..99d17b06b6 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml @@ -62,6 +62,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -678,6 +685,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "server.port") - delete_key(attributes, "url.scheme") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/iis_1: metric_statements: - context: metric @@ -691,6 +815,69 @@ processors: statements: - set(name, "agent.googleapis.com/iis") - set(version, "1.0") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1223,6 +1410,16 @@ processors: statements: - extract_count_metric(true) where name == "grpc.client.attempt.duration" receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1248,6 +1445,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1345,6 +1548,27 @@ service: extensions: - file_storage pipelines: + logs/healthchecks: + exporters: + - googlecloud/logging + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - googlecloud/logging + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - googlecloud/logging diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel_otlp_exporter.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel_otlp_exporter.yaml index 52996669c9..9b251cbbb2 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel_otlp_exporter.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel_otlp_exporter.yaml @@ -69,6 +69,13 @@ processors: match_type: strict metric_names: - fluentbit_uptime + filter/health_checks_1: + error_mode: ignore + logs: + log_record: + - body == nil + - ((body != nil) and body["severity"] == nil) + - ((body != nil and body["severity"] != nil) and (not IsMatch(body["severity"], "INFO|ERROR|WARNING|DEBUG|info|error|warning|debug"))) filter/hostmetrics_1: metrics: exclude: @@ -895,6 +902,123 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/health_checks_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(cache["__parsed_json"], ParseJSON(body["message"])) where (body != nil and body["message"] != nil) + - delete_key(body, "message") where ((body != nil and body["message"] != nil) and (cache != nil and cache["__parsed_json"] != nil)) + - merge_maps(body, cache["__parsed_json"], "upsert") where (cache != nil and cache["__parsed_json"] != nil) + - delete_key(cache, "__parsed_json") where (cache != nil and cache["__parsed_json"] != nil) + - set(cache["__time_valid"], false) + - set(cache["__time_valid"], true) where ((body != nil and body["time"] != nil) and Time(body["time"], "%Y-%m-%dT%H:%M:%S%z") != nil) + - set(time, Time(body["time"], "%Y-%m-%dT%H:%M:%S%z")) where cache["__time_valid"] == true + - delete_key(body, "time") where ((body != nil and body["time"] != nil) and cache["__time_valid"] == true) + - merge_maps(attributes, body["logging.googleapis.com/labels"], "upsert") where body["logging.googleapis.com/labels"] != nil + - delete_key(body, "logging.googleapis.com/labels") where (body != nil and body["logging.googleapis.com/labels"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/httpRequest"]) where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(body, "logging.googleapis.com/httpRequest") where (body != nil and body["logging.googleapis.com/httpRequest"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.http_request"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/logName"]) where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(body, "logging.googleapis.com/logName") where (body != nil and body["logging.googleapis.com/logName"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/operation"]) where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(body, "logging.googleapis.com/operation") where (body != nil and body["logging.googleapis.com/operation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.operation"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/severity"]) where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(body, "logging.googleapis.com/severity") where (body != nil and body["logging.googleapis.com/severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(severity_text, cache["value"]) where (cache != nil and cache["value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/sourceLocation"]) where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(body, "logging.googleapis.com/sourceLocation") where (body != nil and body["logging.googleapis.com/sourceLocation"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(attributes["gcp.source_location"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/spanId"]) where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(body, "logging.googleapis.com/spanId") where (body != nil and body["logging.googleapis.com/spanId"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(span_id.string, cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], body["logging.googleapis.com/trace"]) where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(body, "logging.googleapis.com/trace") where (body != nil and body["logging.googleapis.com/trace"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - delete_key(cache, "__setif_value") where (cache != nil and cache["__setif_value"] != nil) + - set(cache["__setif_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - replace_pattern(cache["__setif_value"], "^projects/([^/]*)/traces/", "") where (cache != nil and cache["__setif_value"] != nil) + - set(trace_id.string, cache["__setif_value"]) where (cache != nil and cache["value"] != nil) + transform/health_checks_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/health_checks_3: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/hostmetrics_3: metric_statements: - context: scope @@ -938,6 +1062,69 @@ processors: - delete_key(attributes, "service.instance.id") - delete_key(attributes, "service.namespace") - delete_key(attributes, "service.version") + transform/logging_ping_0: + error_mode: ignore + log_statements: + - context: log + statements: + - set(observed_time, Now()) + - set(time, Now()) + transform/logging_ping_1: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], body["severity"]) where (body != nil and body["severity"] != nil) + - delete_key(body, "severity") where (body != nil and body["severity"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "ops-agent") + - set(attributes["agent.googleapis.com/health/agentKind"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "latest") + - set(attributes["agent.googleapis.com/health/agentVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], "v1") + - set(attributes["agent.googleapis.com/health/schemaVersion"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - delete_key(cache, "mapped_value") where (cache != nil and cache["mapped_value"] != nil) + - set(cache["mapped_value"], cache["value"]) where (cache != nil and cache["value"] != nil) + - set(cache["mapped_value"], "DEBUG") where cache["value"] == "debug" + - set(cache["mapped_value"], "ERROR") where cache["value"] == "error" + - set(cache["mapped_value"], "INFO") where cache["value"] == "info" + - set(cache["mapped_value"], "WARNING") where cache["value"] == "warn" + - set(severity_text, cache["mapped_value"]) where (cache != nil and cache["mapped_value"] != nil) + - set(severity_number, 0) where (cache != nil and cache["mapped_value"] != nil) + transform/logging_ping_2: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil) + - set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil) + - delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil) + - set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil) + - delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil) + - set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil) + - delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil) + - set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_0"]) where (cache != nil and cache["__field_0"] != nil) + - set(cache["value"], "test-mig") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_1"]) where (cache != nil and cache["__field_1"] != nil) + - set(cache["value"], "test-zone") where cache["value"] == nil + - set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_2"]) where (cache != nil and cache["__field_2"] != nil) + - set(cache["value"], "") where cache["value"] == nil + - set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil) + - delete_key(cache, "value") where (cache != nil and cache["value"] != nil) + - set(cache["value"], cache["__field_3"]) where (cache != nil and cache["__field_3"] != nil) + - set(cache["value"], "ops-agent-health") where cache["value"] == nil + - set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil) transform/logging_windows__event__log_0: error_mode: ignore log_statements: @@ -1515,6 +1702,16 @@ processors: - set(attributes["service.namespace"], resource.attributes["service.namespace"]) where resource.attributes["service.namespace"] != nil - set(attributes["service.instance.id"], resource.attributes["service.instance.id"]) where resource.attributes["service.instance.id"] != nil receivers: + file_log/health_checks: + include: + - health-checks.log + operators: + - from: body + id: body + to: body.message + type: move + start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1540,6 +1737,12 @@ receivers: mute_process_exe_error: true mute_process_name_error: true processes: {} + otlpjsonfile/logging_ping: + include: + - logging_ping_otlp.json + poll_interval: 10m0s + replay_file: true + start_at: beginning otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -1638,6 +1841,35 @@ service: - file_storage - googleclientauth pipelines: + logs/healthchecks: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/health_checks_0 + - filter/health_checks_1 + - transform/health_checks_2 + - transform/health_checks_3 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - file_log/health_checks + logs/loggingping: + exporters: + - otlp_grpc/otlp_logs + processors: + - transform/logging_ping_0 + - transform/logging_ping_1 + - transform/logging_ping_2 + - resourcedetection/_global_0 + - resource/otlp_grpc/otlp_logs_logs_0 + - resource/otlp_grpc/otlp_logs_logs_1 + - transform/otlp_grpc/otlp_logs_logs_2 + - transform/otlp_grpc/otlp_logs_logs_3 + receivers: + - otlpjsonfile/logging_ping logs/logs_default__pipeline_logging_windows__event__log: exporters: - otlp_grpc/otlp_logs diff --git a/integration_test/ops_agent_test/main_test.go b/integration_test/ops_agent_test/main_test.go index 4c03c36eb7..2400c919a4 100644 --- a/integration_test/ops_agent_test/main_test.go +++ b/integration_test/ops_agent_test/main_test.go @@ -4788,7 +4788,7 @@ metrics: } func isHealthCheckTestImage(imageSpec string) bool { - return strings.HasSuffix(imageSpec, "windows-2022") || strings.HasSuffix(imageSpec, "debian-12") + return strings.HasSuffix(imageSpec, "windows-2022") || strings.HasSuffix(imageSpec, "debian-13") } func healthCheckResultMessage(name string, result string, code string) string { @@ -4867,7 +4867,7 @@ func listenToPortForImage(vm *gce.VM) string { return "nohup nc -l -p 20201 1>/dev/null 2>/dev/null &" } -func TestPortsAndAPIHealthChecks(t *testing.T) { +func TestPortsHealthChecks(t *testing.T) { t.Parallel() gce.RunForEachImage(t, func(t *testing.T, imageSpec string) { t.Parallel() @@ -4875,12 +4875,7 @@ func TestPortsAndAPIHealthChecks(t *testing.T) { t.SkipNow() } - customScopes := strings.Join([]string{ - "https://www.googleapis.com/auth/monitoring.read", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/devstorage.read_write", - }, ",") - ctx, dirLog, vm := agents.CommonSetupWithExtraCreateArguments(t, imageSpec, []string{"--scopes", customScopes}) + ctx, dirLog, vm := agents.CommonSetup(t, imageSpec) logger := dirLog.ToMainLog() if !gce.IsWindows(vm.ImageSpec) { @@ -4915,6 +4910,37 @@ func TestPortsAndAPIHealthChecks(t *testing.T) { checkExpectedHealthCheckResult(t, cmdOut, "Network", "PASS", "") checkExpectedHealthCheckResult(t, cmdOut, "Ports", "FAIL", "OtelMetricsPortErr") + checkExpectedHealthCheckResult(t, cmdOut, "API", "PASS", "") + }) +} + +func TestAPIHealthChecks(t *testing.T) { + t.Parallel() + gce.RunForEachImage(t, func(t *testing.T, imageSpec string) { + t.Parallel() + if !isHealthCheckTestImage(imageSpec) || gce.IsOpsAgentUAPPlugin() { + t.SkipNow() + } + + customScopes := strings.Join([]string{ + "https://www.googleapis.com/auth/monitoring.read", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/devstorage.read_write", + }, ",") + ctx, dirLog, vm := agents.CommonSetupWithExtraCreateArguments(t, imageSpec, []string{"--scopes", customScopes}) + logger := dirLog.ToMainLog() + + if err := agents.SetupOpsAgent(ctx, logger, vm, ""); err != nil { + t.Fatal(err) + } + + cmdOut, err := getHealthCheckResultsForImage(ctx, logger, vm) + if err != nil { + t.Fatal(err) + } + + checkExpectedHealthCheckResult(t, cmdOut, "Network", "PASS", "") + checkExpectedHealthCheckResult(t, cmdOut, "Ports", "PASS", "") checkExpectedHealthCheckResult(t, cmdOut, "API", "FAIL", "MonApiScopeErr") query := fmt.Sprintf(`severity="ERROR" AND jsonPayload.code="MonApiScopeErr" AND labels."agent.googleapis.com/health/agentKind"="ops-agent" AND labels."agent.googleapis.com/health/agentVersion"=~"^\d+\.\d+\.\d+.*$" AND labels."agent.googleapis.com/health/schemaVersion"="v1"`) diff --git a/internal/self_metrics/self_metrics.go b/internal/self_metrics/self_metrics.go index 11cf45b0b6..58335dfc91 100644 --- a/internal/self_metrics/self_metrics.go +++ b/internal/self_metrics/self_metrics.go @@ -24,6 +24,7 @@ import ( mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/contrib/detectors/gcp" "go.opentelemetry.io/otel/attribute" @@ -255,6 +256,15 @@ func CollectOpsAgentSelfMetrics(ctx context.Context, userUc, mergedUc *confgener } } +func logToJson(logs plog.Logs) ([]byte, error) { + jsonMarshaler := &plog.JSONMarshaler{} + jsonResult, err := jsonMarshaler.MarshalLogs(logs) + if err != nil { + return nil, err + } + return jsonResult, nil +} + func metricToJson(metrics pmetric.Metrics) ([]byte, error) { jsonMarshaler := &pmetric.JSONMarshaler{} jsonResult, err := jsonMarshaler.MarshalMetrics(metrics) @@ -332,6 +342,20 @@ func CollectFeatureTrackingMetricToOTLPJSON(ctx context.Context, userUc, mergedU return metricToJson(metrics) } +func CollectLoggingPingToOTLPJSON() ([]byte, error) { + logs := plog.NewLogs() + resource := logs.ResourceLogs().AppendEmpty() + resource.Resource().Attributes().PutStr("k", "v") // Resources can't be empty + + logRecord := resource.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + bodyMap := logRecord.Body().SetEmptyMap() + bodyMap.PutStr("code", "LogPingOpsAgent") + bodyMap.PutStr("severity", "DEBUG") + + return logToJson(logs) +} + // config and merged config respectively func getUserAndMergedConfigs(ctx context.Context, userConfPath string) (*confgenerator.UnifiedConfig, *confgenerator.UnifiedConfig, error) { userUc, err := confgenerator.ReadUnifiedConfigFromFile(ctx, userConfPath) @@ -371,5 +395,13 @@ func GenerateOpsAgentSelfMetricsOTLPJSON(ctx context.Context, config, outDir str if err = confgenerator.WriteConfigFile(enabledReceiverOTLPJSON, filepath.Join(outDir, "enabled_receivers_otlp.json")); err != nil { return fmt.Errorf("failed to write enabled receivers metric otlp json file: %w", err) } + + loggingPingOTLPJSON, err := CollectLoggingPingToOTLPJSON() + if err != nil { + return fmt.Errorf("failed to generate logging ping otlp json: %w", err) + } + if err := confgenerator.WriteConfigFile(loggingPingOTLPJSON, filepath.Join(outDir, "logging_ping_otlp.json")); err != nil { + return fmt.Errorf("failed to write logging ping otlp json file: %w", err) + } return nil }